Merge remote-tracking branch 'upstream/master' into debugging-inclusion

Consensys · Oct 23, 2024 · 61b25de · 61b25de
2 parents ec2c8f0 + 21efe6e
commit 61b25de
Show file tree

Hide file tree

Showing 279 changed files with 5,290 additions and 2,020 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -363,7 +363,7 @@ jobs:
           path: build/test-results
 
   acceptanceTests:
-    parallelism: 5
+    parallelism: 4
     executor: machine_large_executor_amd64
     steps:
       - install_java_21

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,5 +9,8 @@
 ### Breaking Changes
 
 ### Additions and Improvements
+- Clean up old beacon states when switching from ARCHIVE to PRUNE or MINIMAL data storage mode
 
 ### Bug Fixes
+ - Fixed a block production issue for Validator Client (24.10.0 to 24.10.2 teku VC), where required headers were not provided for JSON payloads. Default SSZ block production was unaffected.
+ - Block production now uses json data (more like 24.8.0 did than 24.10) if the Eth-Consensus-version header is absent.
diff --git a/acceptance-tests/build.gradle b/acceptance-tests/build.gradle
@@ -28,7 +28,7 @@ dependencies {
   testFixturesImplementation 'io.libp2p:jvm-libp2p'
   testFixturesImplementation 'org.apache.commons:commons-lang3'
   testFixturesImplementation 'commons-io:commons-io'
-  testFixturesImplementation 'org.apache.tuweni:tuweni-bytes'
+  testFixturesImplementation 'io.tmio:tuweni-bytes'
   testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api'
   testFixturesImplementation 'org.testcontainers:testcontainers'
   testFixturesImplementation 'org.testcontainers:junit-jupiter'

diff --git a/...c/acceptance-test/java/tech/pegasys/teku/test/acceptance/BlockProposalAcceptanceTest.java b/...c/acceptance-test/java/tech/pegasys/teku/test/acceptance/BlockProposalAcceptanceTest.java
@@ -21,7 +21,8 @@
 import java.util.Arrays;
 import java.util.Locale;
 import org.apache.tuweni.bytes.Bytes32;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 import tech.pegasys.teku.infrastructure.bytes.Bytes20;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.test.acceptance.dsl.AcceptanceTestBase;
@@ -34,8 +35,9 @@
 public class BlockProposalAcceptanceTest extends AcceptanceTestBase {
   private static final URL JWT_FILE = Resources.getResource("auth/ee-jwt-secret.hex");
 
-  @Test
-  void shouldHaveCorrectFeeRecipientAndGraffiti() throws Exception {
+  @ParameterizedTest(name = "ssz_encode={0}")
+  @ValueSource(booleans = {true, false})
+  void shouldHaveCorrectFeeRecipientAndGraffiti(final boolean useSszBlocks) throws Exception {
     final String networkName = "swift";
 
     final ValidatorKeystores validatorKeystores =
@@ -69,6 +71,7 @@ void shouldHaveCorrectFeeRecipientAndGraffiti() throws Exception {
                 .withValidatorProposerDefaultFeeRecipient(defaultFeeRecipient)
                 .withInteropModeDisabled()
                 .withBeaconNodes(beaconNode)
+                .withBeaconNodeSszBlocksEnabled(useSszBlocks)
                 .withGraffiti(userGraffiti)
                 .withNetwork("auto")
                 .build());

diff --git a/...est/java/tech/pegasys/teku/test/acceptance/ExecutionLayerTriggeredExitAcceptanceTest.java b/...est/java/tech/pegasys/teku/test/acceptance/ExecutionLayerTriggeredExitAcceptanceTest.java
@@ -78,9 +78,9 @@ void triggerValidatorExitWithFullWithdrawal() throws Exception {
     tekuNode.waitForNewFinalization();
 
     final ValidatorKeys validator = validatorKeys.getValidatorKeys().get(0);
-    final BLSPublicKey validatorPublicKey = validator.getValidatorKey().getPublicKey();
+    final BLSPublicKey validatorPubkey = validator.getValidatorKey().getPublicKey();
 
-    besuNode.createWithdrawalRequest(eth1PrivateKey, validatorPublicKey, UInt64.ZERO);
+    besuNode.createWithdrawalRequest(eth1PrivateKey, validatorPubkey, UInt64.ZERO);
 
     // Wait for validator exit confirmation
     tekuNode.waitForLogMessageContaining(

diff --git a/...nce-test/java/tech/pegasys/teku/test/acceptance/ValidatorConsolidationAcceptanceTest.java b/...nce-test/java/tech/pegasys/teku/test/acceptance/ValidatorConsolidationAcceptanceTest.java
@@ -80,19 +80,19 @@ void consolidateValidator() throws Exception {
     tekuNode.waitForNewFinalization();
 
     final ValidatorKeys sourceValidator = validatorKeys.getValidatorKeys().get(0);
-    final BLSPublicKey sourceValidatorPublicKey = sourceValidator.getValidatorKey().getPublicKey();
+    final BLSPublicKey sourceValidatorPubkey = sourceValidator.getValidatorKey().getPublicKey();
 
     final ValidatorKeys targetValidator = validatorKeys.getValidatorKeys().get(1);
-    final BLSPublicKey targetValidatorPublicKey = targetValidator.getValidatorKey().getPublicKey();
+    final BLSPublicKey targetValidatorPubkey = targetValidator.getValidatorKey().getPublicKey();
 
     besuNode.createConsolidationRequest(
-        eth1PrivateKey, sourceValidatorPublicKey, targetValidatorPublicKey);
-    waitForValidatorExit(tekuNode, sourceValidatorPublicKey);
+        eth1PrivateKey, sourceValidatorPubkey, targetValidatorPubkey);
+    waitForValidatorExit(tekuNode, sourceValidatorPubkey);
   }
 
   private void waitForValidatorExit(
-      final TekuBeaconNode tekuNode, final BLSPublicKey validatorPublicKey) {
-    final String pubKeySubstring = validatorPublicKey.toHexString().substring(2, 9);
+      final TekuBeaconNode tekuNode, final BLSPublicKey validatorPubkey) {
+    final String pubKeySubstring = validatorPubkey.toHexString().substring(2, 9);
     tekuNode.waitForLogMessageContaining(
         "Validator "
             + pubKeySubstring

diff --git a/...ys/teku/test/acceptance/validatorslashing/MultiPeersStandAloneVcBlocksAcceptanceTest.java b/...ys/teku/test/acceptance/validatorslashing/MultiPeersStandAloneVcBlocksAcceptanceTest.java
@@ -17,7 +17,6 @@
 import org.junit.jupiter.params.provider.MethodSource;
 import tech.pegasys.teku.bls.BLSKeyPair;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
-import tech.pegasys.teku.spec.SpecMilestone;
 import tech.pegasys.teku.test.acceptance.dsl.TekuBeaconNode;
 import tech.pegasys.teku.test.acceptance.dsl.TekuNodeConfigBuilder;
 import tech.pegasys.teku.test.acceptance.dsl.TekuValidatorNode;
@@ -39,43 +38,25 @@ public class MultiPeersStandAloneVcBlocksAcceptanceTest
       shouldShutDownWhenOwnedValidatorSlashed_StandAloneVC_MultiplePeers_SlashingThroughBlock_NoSlashingEventsGossip(
           final SlashingEventType slashingEventType) throws Exception {
 
-    final int genesisTime = timeProvider.getTimeInSeconds().plus(10).intValue();
-    final UInt64 altairEpoch = UInt64.valueOf(100);
+    final int genesisTime = timeProvider.getTimeInSeconds().plus(30).intValue();
 
     final TekuBeaconNode firstTekuNode =
         createTekuBeaconNode(
             TekuNodeConfigBuilder.createBeaconNode()
                 .withGenesisTime(genesisTime)
                 .withNetwork(network)
                 .withRealNetwork()
-                .withAltairEpoch(altairEpoch)
+                .withSubscribeAllSubnetsEnabled()
                 .withInteropValidators(0, 32)
                 .build());
 
-    firstTekuNode.start();
-
-    firstTekuNode.waitForEpochAtOrAbove(2);
-
-    final int slashedValidatorIndex = 34;
-    final BLSKeyPair slashedValidatorKeyPair = getBlsKeyPair(slashedValidatorIndex);
-    final int slotInThirdEpoch =
-        firstTekuNode.getSpec().forMilestone(SpecMilestone.ALTAIR).getSlotsPerEpoch() * 2 + 3;
-
-    postSlashing(
-        firstTekuNode,
-        UInt64.valueOf(slotInThirdEpoch),
-        UInt64.valueOf(slashedValidatorIndex),
-        slashedValidatorKeyPair.getSecretKey(),
-        slashingEventType);
-
     final TekuBeaconNode secondBeaconNode =
         createTekuBeaconNode(
             TekuNodeConfigBuilder.createBeaconNode()
                 .withGenesisTime(genesisTime)
                 .withNetwork(network)
                 .withRealNetwork()
-                .withRealNetwork()
-                .withAltairEpoch(altairEpoch)
+                .withSubscribeAllSubnetsEnabled()
                 .withPeers(firstTekuNode)
                 .build());
 
@@ -89,10 +70,23 @@ public class MultiPeersStandAloneVcBlocksAcceptanceTest
                 .withBeaconNodes(secondBeaconNode)
                 .build());
 
+    firstTekuNode.start();
     secondBeaconNode.start();
-
     secondValidatorClient.start();
 
+    firstTekuNode.waitForEpochAtOrAbove(1);
+
+    final int slashedValidatorIndex = 34;
+    final BLSKeyPair slashedValidatorKeyPair = getBlsKeyPair(slashedValidatorIndex);
+    final int slotInSecondEpoch = firstTekuNode.getSpec().getGenesisSpec().getSlotsPerEpoch() + 3;
+
+    postSlashing(
+        firstTekuNode,
+        UInt64.valueOf(slotInSecondEpoch),
+        UInt64.valueOf(slashedValidatorIndex),
+        slashedValidatorKeyPair.getSecretKey(),
+        slashingEventType);
+
     secondValidatorClient.waitForLogMessageContaining(
         String.format(slashingActionLog, slashedValidatorKeyPair.getPublicKey().toHexString()));
 

diff --git a/...asys/teku/test/acceptance/validatorslashing/ValidatorSlashingDetectionAcceptanceTest.java b/...asys/teku/test/acceptance/validatorslashing/ValidatorSlashingDetectionAcceptanceTest.java
@@ -47,7 +47,7 @@ public class ValidatorSlashingDetectionAcceptanceTest extends AcceptanceTestBase
   final String slashingActionLog =
       "Validator slashing detection is enabled and validator(s) with public key(s) %s detected as slashed. "
           + "Shutting down...";
-  final int shutdownWaitingSeconds = 60;
+  final int shutdownWaitingSeconds = 90;
 
   enum SlashingEventType {
     PROPOSER_SLASHING,

diff --git a/...nce-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuBeaconNode.java b/...nce-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuBeaconNode.java
@@ -226,6 +226,7 @@ public void postProposerSlashing(
         randomSignedBeaconBlockHeader(slot, index, secretKey, signingRootUtil, forkInfo);
     final SignedBeaconBlockHeader header2 =
         randomSignedBeaconBlockHeader(slot, index, secretKey, signingRootUtil, forkInfo);
+    LOG.debug("Inserting proposer slashing for index {} at slot {}", index, slot);
     final String body =
         JsonUtil.serialize(
             new ProposerSlashing(header1, header2),
@@ -256,6 +257,7 @@ public void postAttesterSlashing(
         spec.getGenesisSchemaDefinitions()
             .getAttesterSlashingSchema()
             .create(indexedAttestation1, indexedAttestation2);
+    LOG.debug("Inserting attester slashing for index {} at slot {}", slashedIndex, slashingSlot);
     final String body =
         JsonUtil.serialize(
             attesterSlashing,
@@ -282,7 +284,7 @@ private IndexedAttestation randomIndexedAttestation(
             secretKey,
             signingRootUtil.signingRootForSignAttestationData(attestationData, forkInfo));
 
-    final IndexedAttestationSchema<?> schema =
+    final IndexedAttestationSchema schema =
         spec.getGenesisSchemaDefinitions().getIndexedAttestationSchema();
     return schema.create(
         Stream.of(index).collect(schema.getAttestingIndicesSchema().collectorUnboxed()),

diff --git a/...ts/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNodeConfigBuilder.java b/...ts/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNodeConfigBuilder.java
@@ -14,6 +14,7 @@
 package tech.pegasys.teku.test.acceptance.dsl;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.assertj.core.api.Assertions.assertThat;
 import static tech.pegasys.teku.test.acceptance.dsl.Node.DATA_PATH;
 import static tech.pegasys.teku.test.acceptance.dsl.Node.JWT_SECRET_FILE_PATH;
 import static tech.pegasys.teku.test.acceptance.dsl.Node.METRICS_PORT;
@@ -176,7 +177,7 @@ public TekuNodeConfigBuilder withTrustedSetupFromClasspath(final String trustedS
     LOG.debug("Xtrusted-setup={}", TRUSTED_SETUP_FILE);
     configMap.put("Xtrusted-setup", TRUSTED_SETUP_FILE);
     final URL trustedSetupResource = Eth2NetworkConfiguration.class.getResource(trustedSetup);
-    assert trustedSetupResource != null;
+    assertThat(trustedSetupResource).isNotNull();
     configFileMap.put(copyToTmpFile(trustedSetupResource), TRUSTED_SETUP_FILE);
     return this;
   }
@@ -417,6 +418,12 @@ public TekuNodeConfigBuilder withDoppelgangerDetectionEnabled() {
     return this;
   }
 
+  public TekuNodeConfigBuilder withSubscribeAllSubnetsEnabled() {
+    LOG.debug("p2p-subscribe-all-subnets-enabled=true");
+    configMap.put("p2p-subscribe-all-subnets-enabled", true);
+    return this;
+  }
+
   public TekuNodeConfigBuilder withDepositsFrom(final BesuNode eth1Node) {
     mustBe(NodeType.BEACON_NODE);
     configMap.put("Xinterop-enabled", false);
@@ -524,6 +531,12 @@ public TekuNodeConfigBuilder withValidatorProposerDefaultFeeRecipient(
     return this;
   }
 
+  public TekuNodeConfigBuilder withBeaconNodeSszBlocksEnabled(final boolean enabled) {
+    LOG.debug("beacon-node-ssz-blocks-enabled={}", enabled);
+    configMap.put("beacon-node-ssz-blocks-enabled", enabled);
+    return this;
+  }
+
   public TekuNodeConfigBuilder withStartupTargetPeerCount(final int startupTargetPeerCount) {
     mustBe(NodeType.BEACON_NODE);
     LOG.debug("Xstartup-target-peer-count={}", startupTargetPeerCount);

diff --git a/...ava/tech/pegasys/teku/test/acceptance/dsl/executionrequests/ExecutionRequestsService.java b/...ava/tech/pegasys/teku/test/acceptance/dsl/executionrequests/ExecutionRequestsService.java
@@ -102,15 +102,15 @@ public SafeFuture<TransactionReceipt> createWithdrawalRequest(
   }
 
   public SafeFuture<TransactionReceipt> createConsolidationRequest(
-      final BLSPublicKey sourceValidatorPublicKey, final BLSPublicKey targetValidatorPublicKey) {
+      final BLSPublicKey sourceValidatorPubkey, final BLSPublicKey targetValidatorPubkey) {
     // Sanity check that we can interact with the contract
     Waiter.waitFor(
         () ->
             assertThat(consolidationRequestContract.getExcessConsolidationRequests().get())
                 .isEqualTo(0));
 
     return consolidationRequestContract
-        .createConsolidationRequest(sourceValidatorPublicKey, targetValidatorPublicKey)
+        .createConsolidationRequest(sourceValidatorPubkey, targetValidatorPubkey)
         .thenCompose(
             response -> {
               final String txHash = response.getResult();

diff --git a/beacon/pow/build.gradle b/beacon/pow/build.gradle
@@ -18,7 +18,7 @@ dependencies {
 
   api 'org.web3j:core'
 
-  implementation 'org.apache.tuweni:tuweni-units'
+  implementation 'io.tmio:tuweni-units'
 
   testImplementation testFixtures(project(':infrastructure:async'))
   testImplementation testFixtures(project(':infrastructure:time'))

diff --git a/beacon/sync/build.gradle b/beacon/sync/build.gradle
@@ -16,7 +16,7 @@ dependencies {
     implementation project(':storage:api')
     implementation project(':infrastructure:events')
 
-    implementation 'org.apache.tuweni:tuweni-bytes'
+    implementation 'io.tmio:tuweni-bytes'
 
     testImplementation testFixtures(project(':ethereum:spec'))
     testImplementation testFixtures(project(':ethereum:statetransition'))

diff --git a/beacon/validator/build.gradle b/beacon/validator/build.gradle
@@ -33,8 +33,8 @@ dependencies {
   implementation project(':ethereum:json-types')
 
   implementation 'it.unimi.dsi:fastutil'
-  implementation 'org.apache.tuweni:tuweni-bytes'
-  implementation 'org.apache.tuweni:tuweni-ssz'
+  implementation 'io.tmio:tuweni-bytes'
+  implementation 'io.tmio:tuweni-ssz'
 
   testImplementation project(':infrastructure:metrics')
   testImplementation testFixtures(project(':ethereum:spec'))

diff --git a/.../src/main/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactory.java b/.../src/main/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactory.java
@@ -49,7 +49,7 @@
 import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadContext;
 import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadHeader;
 import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadResult;
-import tech.pegasys.teku.spec.datastructures.execution.versions.electra.ExecutionRequestsBuilderElectra;
+import tech.pegasys.teku.spec.datastructures.execution.versions.electra.ExecutionRequests;
 import tech.pegasys.teku.spec.datastructures.operations.Attestation;
 import tech.pegasys.teku.spec.datastructures.operations.AttesterSlashing;
 import tech.pegasys.teku.spec.datastructures.operations.ProposerSlashing;
@@ -64,7 +64,6 @@
 import tech.pegasys.teku.spec.schemas.SchemaDefinitions;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsBellatrix;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;
-import tech.pegasys.teku.spec.schemas.SchemaDefinitionsElectra;
 import tech.pegasys.teku.statetransition.OperationPool;
 import tech.pegasys.teku.statetransition.attestation.AggregatingAttestationPool;
 import tech.pegasys.teku.statetransition.attestation.AttestationForkChecker;
@@ -240,19 +239,11 @@ private SafeFuture<Void> setExecutionData(
             requestedBuilderBoostFactor,
             blockProductionPerformance);
 
-    // TODO Update as part of Electra Engine API updates
-    // (https://github.com/Consensys/teku/issues/8620)
-    if (bodyBuilder.supportsExecutionRequests()) {
-      bodyBuilder.executionRequests(
-          new ExecutionRequestsBuilderElectra(
-                  SchemaDefinitionsElectra.required(schemaDefinitions).getExecutionRequestsSchema())
-              .build());
-    }
-
     return SafeFuture.allOf(
         cacheExecutionPayloadValue(executionPayloadResult, blockSlotState),
         setPayloadOrPayloadHeader(bodyBuilder, executionPayloadResult),
-        setKzgCommitments(bodyBuilder, schemaDefinitions, executionPayloadResult));
+        setKzgCommitments(bodyBuilder, schemaDefinitions, executionPayloadResult),
+        setExecutionRequests(bodyBuilder, executionPayloadResult));
   }
 
   private SafeFuture<Void> cacheExecutionPayloadValue(
@@ -350,6 +341,43 @@ private SszList<SszKZGCommitment> getBlobKzgCommitmentsFromBuilderFlow(
         .orElseThrow();
   }
 
+  private SafeFuture<Void> setExecutionRequests(
+      final BeaconBlockBodyBuilder bodyBuilder,
+      final ExecutionPayloadResult executionPayloadResult) {
+    if (!bodyBuilder.supportsExecutionRequests()) {
+      return SafeFuture.COMPLETE;
+    }
+    final SafeFuture<ExecutionRequests> executionRequests;
+    if (executionPayloadResult.isFromLocalFlow()) {
+      // local, non-blinded flow
+      executionRequests =
+          executionPayloadResult
+              .getExecutionRequestsFutureFromLocalFlow()
+              .orElseThrow()
+              .thenApply(Optional::orElseThrow);
+    } else {
+      // builder, blinded flow
+      executionRequests =
+          executionPayloadResult
+              .getBuilderBidOrFallbackDataFuture()
+              .orElseThrow()
+              .thenApply(this::getExecutionRequestsFromBuilderFlow);
+    }
+
+    return executionRequests.thenAccept(bodyBuilder::executionRequests);
+  }
+
+  private ExecutionRequests getExecutionRequestsFromBuilderFlow(
+      final BuilderBidOrFallbackData builderBidOrFallbackData) {
+    return builderBidOrFallbackData
+        .getBuilderBid()
+        // from the builder bid
+        .flatMap(BuilderBid::getOptionalExecutionRequests)
+        // from the local fallback
+        .or(() -> builderBidOrFallbackData.getFallbackDataRequired().getExecutionRequests())
+        .orElseThrow();
+  }
+
   public Consumer<SignedBeaconBlockUnblinder> createBlockUnblinderSelector(
       final BlockPublishingPerformance blockPublishingPerformance) {
     return bodyUnblinder -> {
@@ -437,7 +465,7 @@ public Function<SignedBlockContainer, List<BlobSidecar>> createBlobSidecarsSelec
         // the blobs and the proofs wouldn't be part of the BlockContainer.
         final BuilderPayloadOrFallbackData builderPayloadOrFallbackData =
             executionLayerBlockProductionManager
-                .getCachedUnblindedPayload(slot)
+                .getCachedUnblindedPayload(block.getSlotAndBlockRoot())
                 .orElseThrow(
                     () ->
                         new IllegalStateException(