Merge branch 'main' into zkbesu

# Conflicts:
#	build.gradle

.github/ISSUE_TEMPLATE/release-checklist.md

@@ -29,9 +29,10 @@ assignees: ''
     - publishes the docker `latest` tag variants
 - [ ] Check binary SHAs are correct on the release page
 - [ ] Check "Container Verify" GitHub workflow has run successfully
-- [ ] Create besu-docs release - https://github.com/hyperledger/besu-docs/releases/new
-  - Copy release notes from besu
-  - If publishing the release in github doesn't automatically trigger this workflow, then manually run https://github.com/hyperledger/besu-docs/actions/workflows/update-version.yml
-- [ ] Create homebrew release - run GHA workflow directly https://github.com/hyperledger/homebrew-besu/actions/workflows/update-version.yml
+- [ ] Update the besu-docs version [update-version workflow](https://github.com/hyperledger/besu-docs/actions/workflows/update-version.yml)
+  - If the PR has not been automatically created, create the PR manually using the created branch `besu-version-<version>`
+- [ ] Create homebrew release using [update-version workflow](https://github.com/hyperledger/homebrew-besu/actions/workflows/update-version.yml)
+  - If the PR has not been automatically created, create the PR manually using the created branch `update-<version>`
+  - Run commands `brew tap hyperledger/besu && brew install besu` on MacOSX and verify latest version has been installed
 - [ ] Delete the burn-in nodes (unless required for further analysis eg performance)
 - [ ] Social announcements

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [Unreleased]
+
+### Fixed
+- **DebugMetrics**: Fixed a `ClassCastException` occurring in `DebugMetrics` when handling nested metric structures. Previously, `Double` values within these structures were incorrectly cast to `Map` objects, leading to errors. This update allows for proper handling of both direct values and nested structures at the same level. Issue# [#7383](https://github.com/hyperledger/besu/pull/7383)
+- `evmtool` was not respecting the `--genesis` setting, resulting in unexpected trace results. [#7433](https://github.com/hyperledger/besu/pull/7433)
+
+### Tests
+- Added a comprehensive test case to reproduce the bug and verify the fix for the `ClassCastException` in `DebugMetrics`. This ensures that complex, dynamically nested metric structures can be handled without errors.
+
 ## Next release
 
 ### Upcoming Breaking Changes
@@ -11,12 +20,14 @@
 - Add 'inbound' field to admin_peers JSON-RPC Call [#7461](https://github.com/hyperledger/besu/pull/7461)
 - Add pending block header to `TransactionEvaluationContext` plugin API [#7483](https://github.com/hyperledger/besu/pull/7483)
 - Add bootnode to holesky config [#7500](https://github.com/hyperledger/besu/pull/7500)
+- Implement engine_getClientVersionV1
 
 ### Bug fixes
 - Fix tracing in precompiled contracts when halting for out of gas [#7318](https://github.com/hyperledger/besu/issues/7318)
 - Correctly release txpool save and restore lock in case of exceptions [#7473](https://github.com/hyperledger/besu/pull/7473)
 - Fix for `eth_gasPrice` could not retrieve block error [#7482](https://github.com/hyperledger/besu/pull/7482)
 
+
 ## 24.8.0
 
 ### Upcoming Breaking Changes
@@ -41,6 +52,7 @@
 - Correct entrypoint in Docker evmtool [#7430](https://github.com/hyperledger/besu/pull/7430)
 - Fix protocol schedule check for devnets [#7429](https://github.com/hyperledger/besu/pull/7429)
 - Fix behaviour when starting in a pre-merge network [#7431](https://github.com/hyperledger/besu/pull/7431)
+- Fix Null pointer from DNS daemon [#7505](https://github.com/hyperledger/besu/issues/7505)
 
 ## 24.7.1
 

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java

@@ -32,7 +32,9 @@
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.permissioning.LocalPermissioningConfiguration;
 import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
+import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
 import org.hyperledger.besu.pki.keystore.KeyStoreWrapper;
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode;
 import org.hyperledger.besu.tests.acceptance.dsl.node.Node;
 import org.hyperledger.besu.tests.acceptance.dsl.node.RunnableNode;
@@ -476,21 +478,28 @@ public BesuNode createIbft2Node(final String name, final String genesisFile) thr
             .build());
   }
 
-  public BesuNode createIbft2Node(final String name, final boolean fixedPort) throws IOException {
+  public BesuNode createIbft2Node(
+      final String name, final boolean fixedPort, final DataStorageFormat storageFormat)
+      throws IOException {
     JsonRpcConfiguration rpcConfig = node.createJsonRpcWithIbft2EnabledConfig(false);
     rpcConfig.addRpcApi("ADMIN,TXPOOL");
     if (fixedPort) {
       rpcConfig.setPort(
           Math.abs(name.hashCode() % 60000)
               + 1024); // Generate a consistent port for p2p based on node name
     }
+
     BesuNodeConfigurationBuilder builder =
         new BesuNodeConfigurationBuilder()
             .name(name)
             .miningEnabled()
             .jsonRpcConfiguration(rpcConfig)
             .webSocketConfiguration(node.createWebSocketEnabledConfig())
             .devMode(false)
+            .dataStorageConfiguration(
+                storageFormat == DataStorageFormat.FOREST
+                    ? DataStorageConfiguration.DEFAULT_FOREST_CONFIG
+                    : DataStorageConfiguration.DEFAULT_BONSAI_CONFIG)
             .genesisConfigProvider(GenesisConfigurationFactory::createIbft2GenesisConfig);
     if (fixedPort) {
       builder.p2pPort(
@@ -527,7 +536,9 @@ public BesuNode createQbftNodeWithTLSPKCS11(final String name) throws IOExceptio
     return createQbftNodeWithTLS(name, KeyStoreWrapper.KEYSTORE_TYPE_PKCS11);
   }
 
-  public BesuNode createQbftNode(final String name, final boolean fixedPort) throws IOException {
+  public BesuNode createQbftNode(
+      final String name, final boolean fixedPort, final DataStorageFormat storageFormat)
+      throws IOException {
     JsonRpcConfiguration rpcConfig = node.createJsonRpcWithQbftEnabledConfig(false);
     rpcConfig.addRpcApi("ADMIN,TXPOOL");
     if (fixedPort) {
@@ -543,6 +554,10 @@ public BesuNode createQbftNode(final String name, final boolean fixedPort) throw
             .jsonRpcConfiguration(rpcConfig)
             .webSocketConfiguration(node.createWebSocketEnabledConfig())
             .devMode(false)
+            .dataStorageConfiguration(
+                storageFormat == DataStorageFormat.FOREST
+                    ? DataStorageConfiguration.DEFAULT_FOREST_CONFIG
+                    : DataStorageConfiguration.DEFAULT_BONSAI_CONFIG)
             .genesisConfigProvider(GenesisConfigurationFactory::createQbftGenesisConfig);
     if (fixedPort) {
       builder.p2pPort(

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bft/BftAcceptanceTestParameterization.java

@@ -14,6 +14,7 @@
  */
 package org.hyperledger.besu.tests.acceptance.bft;
 
+import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.BesuNodeFactory;
 
@@ -38,7 +39,9 @@ public static Stream<Arguments> getFactories() {
   @FunctionalInterface
   public interface NodeCreator {
 
-    BesuNode create(BesuNodeFactory factory, String name, boolean fixedPort) throws Exception;
+    BesuNode create(
+        BesuNodeFactory factory, String name, boolean fixedPort, DataStorageFormat storageFormat)
+        throws Exception;
   }
 
   @FunctionalInterface
@@ -64,11 +67,15 @@ public BftAcceptanceTestParameterization(
   }
 
   public BesuNode createNode(BesuNodeFactory factory, String name) throws Exception {
-    return creatorFn.create(factory, name, false);
+    return creatorFn.create(factory, name, false, DataStorageFormat.FOREST);
+  }
+
+  public BesuNode createBonsaiNodeFixedPort(BesuNodeFactory factory, String name) throws Exception {
+    return creatorFn.create(factory, name, true, DataStorageFormat.BONSAI);
   }
 
-  public BesuNode createNodeFixedPort(BesuNodeFactory factory, String name) throws Exception {
-    return creatorFn.create(factory, name, true);
+  public BesuNode createForestNodeFixedPort(BesuNodeFactory factory, String name) throws Exception {
+    return creatorFn.create(factory, name, true, DataStorageFormat.FOREST);
   }
 
   public BesuNode createNodeWithValidators(

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bftsoak/BftMiningSoakTest.java

@@ -60,10 +60,11 @@ public void shouldBeStableDuringLongTest(
     // in between certain steps. There should be no upper-limit to how long the test is run for
     assertThat(getTestDurationMins()).isGreaterThanOrEqualTo(MIN_TEST_TIME_MINS);
 
-    final BesuNode minerNode1 = nodeFactory.createNodeFixedPort(besu, "miner1");
-    final BesuNode minerNode2 = nodeFactory.createNodeFixedPort(besu, "miner2");
-    final BesuNode minerNode3 = nodeFactory.createNodeFixedPort(besu, "miner3");
-    final BesuNode minerNode4 = nodeFactory.createNodeFixedPort(besu, "miner4");
+    // Create a mix of Bonsai and Forest DB nodes
+    final BesuNode minerNode1 = nodeFactory.createBonsaiNodeFixedPort(besu, "miner1");
+    final BesuNode minerNode2 = nodeFactory.createForestNodeFixedPort(besu, "miner2");
+    final BesuNode minerNode3 = nodeFactory.createBonsaiNodeFixedPort(besu, "miner3");
+    final BesuNode minerNode4 = nodeFactory.createForestNodeFixedPort(besu, "miner4");
 
     // Each step should be given a minimum of 3 minutes to complete successfully. If the time
     // give to run the soak test results in a time-per-step lower than this then the time

besu/src/main/java/org/hyperledger/besu/BesuInfo.java

@@ -17,16 +17,38 @@
 import org.hyperledger.besu.util.platform.PlatformDetector;
 
 import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Represent Besu information such as version, OS etc. Used with --version option and during Besu
  * start.
  */
 public final class BesuInfo {
   private static final String CLIENT = "besu";
-  private static final String VERSION = BesuInfo.class.getPackage().getImplementationVersion();
   private static final String OS = PlatformDetector.getOS();
   private static final String VM = PlatformDetector.getVM();
+  private static final String VERSION;
+  private static final String COMMIT;
+
+  static {
+    String projectVersion = BesuInfo.class.getPackage().getImplementationVersion();
+    if (projectVersion == null) {
+      // protect against unset project version (e.g. unit tests being run, etc)
+      VERSION = null;
+      COMMIT = null;
+    } else {
+      Pattern pattern =
+          Pattern.compile("(?<version>\\d+\\.\\d+\\.?\\d?-?\\w*)-(?<commit>[0-9a-fA-F]{8})");
+      Matcher matcher = pattern.matcher(projectVersion);
+      if (matcher.find()) {
+        VERSION = matcher.group("version");
+        COMMIT = matcher.group("commit");
+      } else {
+        throw new RuntimeException("Invalid project version: " + projectVersion);
+      }
+    }
+  }
 
   private BesuInfo() {}
 
@@ -46,7 +68,7 @@ public static String shortVersion() {
    *     or "besu/v23.1.0/osx-aarch_64/corretto-java-19"
    */
   public static String version() {
-    return String.format("%s/v%s/%s/%s", CLIENT, VERSION, OS, VM);
+    return String.format("%s/v%s-%s/%s/%s", CLIENT, VERSION, COMMIT, OS, VM);
   }
 
   /**
@@ -57,7 +79,18 @@ public static String version() {
    */
   public static String nodeName(final Optional<String> maybeIdentity) {
     return maybeIdentity
-        .map(identity -> String.format("%s/%s/v%s/%s/%s", CLIENT, identity, VERSION, OS, VM))
+        .map(
+            identity ->
+                String.format("%s/%s/v%s-%s/%s/%s", CLIENT, identity, VERSION, COMMIT, OS, VM))
         .orElse(version());
   }
+
+  /**
+   * Generate the commit hash for this besu version
+   *
+   * @return the commit hash for this besu version
+   */
+  public static String commit() {
+    return COMMIT;
+  }
 }

besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java

@@ -1291,6 +1291,8 @@ private Map<String, JsonRpcMethod> jsonRpcMethods(
         new JsonRpcMethodsFactory()
             .methods(
                 BesuInfo.nodeName(identityString),
+                BesuInfo.shortVersion(),
+                BesuInfo.commit(),
                 ethNetworkConfig.networkId(),
                 besuController.getGenesisConfigOptions(),
                 network,

besu/src/main/java/org/hyperledger/besu/services/SynchronizationServiceImpl.java

@@ -74,17 +74,11 @@ public SynchronizationServiceImpl(
   @Override
   public void fireNewUnverifiedForkchoiceEvent(
       final Hash head, final Hash safeBlock, final Hash finalizedBlock) {
-    final MergeContext mergeContext = protocolContext.getConsensusContext(MergeContext.class);
-    if (mergeContext != null) {
-      mergeContext.fireNewUnverifiedForkchoiceEvent(head, safeBlock, finalizedBlock);
-      protocolContext.getBlockchain().setFinalized(finalizedBlock);
-      protocolContext.getBlockchain().setSafeBlock(safeBlock);
-    } else {
-      LOG.atWarn()
-          .setMessage(
-              "The merge context is unavailable, hence the fork choice event cannot be triggered")
-          .log();
-    }
+    protocolContext
+        .safeConsensusContext(MergeContext.class)
+        .ifPresent(mc -> mc.fireNewUnverifiedForkchoiceEvent(head, safeBlock, finalizedBlock));
+    protocolContext.getBlockchain().setFinalized(finalizedBlock);
+    protocolContext.getBlockchain().setSafeBlock(safeBlock);
   }
 
   @Override

besu/src/test/java/org/hyperledger/besu/BesuInfoTest.java

@@ -33,7 +33,8 @@ public final class BesuInfoTest {
    */
   @Test
   public void versionStringIsEthstatsFriendly() {
-    assertThat(BesuInfo.version()).matches("[^/]+/v(\\d+\\.\\d+\\.\\d+[^/]*|null)/[^/]+/[^/]+");
+    assertThat(BesuInfo.version())
+        .matches("[^/]+/v(\\d+\\.\\d+\\.\\d+[^/]*|null-null)/[^/]+/[^/]+");
   }
 
   /**
@@ -45,7 +46,7 @@ public void versionStringIsEthstatsFriendly() {
   @Test
   public void noIdentityNodeNameIsEthstatsFriendly() {
     assertThat(BesuInfo.nodeName(Optional.empty()))
-        .matches("[^/]+/v(\\d+\\.\\d+\\.\\d+[^/]*|null)/[^/]+/[^/]+");
+        .matches("[^/]+/v(\\d+\\.\\d+\\.\\d+[^/]*|null-null)/[^/]+/[^/]+");
   }
 
   /**
@@ -58,6 +59,6 @@ public void noIdentityNodeNameIsEthstatsFriendly() {
   @Test
   public void userIdentityNodeNameIsEthstatsFriendly() {
     assertThat(BesuInfo.nodeName(Optional.of("TestUserIdentity")))
-        .matches("[^/]+/[^/]+/v(\\d+\\.\\d+\\.\\d+[^/]*|null)/[^/]+/[^/]+");
+        .matches("[^/]+/[^/]+/v(\\d+\\.\\d+\\.\\d+[^/]*|null-null)/[^/]+/[^/]+");
   }
 }

build.gradle

@@ -32,6 +32,10 @@ plugins {
   id 'maven-publish'
 }
 
+tasks.register('spotlessCheckFast') {
+    dependsOn subprojects.collect { it.tasks.withType(com.diffplug.gradle.spotless.SpotlessCheck) }
+}
+
 if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_21)) {
   throw new GradleException("Java 21 or later is required to build Besu.\n" +
   "  Detected version ${JavaVersion.current()}")
@@ -145,6 +149,10 @@ allprojects {
       url 'https://splunk.jfrog.io/splunk/ext-releases-local'
       content { includeGroupByRegex('com\\.splunk\\..*') }
     }
+    maven {
+      url 'https://gitlab.com/api/v4/projects/19871573/packages/maven'
+      content { includeGroupByRegex('com\\.gitlab\\.javafuzz(\\..*)?') }
+    }
 
     mavenCentral()
 
@@ -415,6 +423,12 @@ allprojects {
     options.addStringOption('Xwerror', '-html5')
     options.encoding = 'UTF-8'
   }
+
+  plugins.withType(JavaPlugin) {
+    tasks.withType(JavaCompile) {
+      it.dependsOn(rootProject.tasks.named('spotlessCheckFast'))
+    }
+  }
 }
 
 task deploy() {}
@@ -438,7 +452,7 @@ task checkMavenCoordinateCollisions {
 
 tasks.register('checkPluginAPIChanges', DefaultTask) {}
 checkPluginAPIChanges.dependsOn(':plugin-api:checkAPIChanges')
-check.dependsOn('checkPluginAPIChanges', 'checkMavenCoordinateCollisions')
+check.dependsOn('checkPluginAPIChanges', 'checkMavenCoordinateCollisions', 'spotlessCheckFast')
 
 subprojects {
 
@@ -790,7 +804,7 @@ task distDocker {
         dockerPlatform = "--platform ${project.getProperty('docker-platform')}"
         println "Building for platform ${project.getProperty('docker-platform')}"
       }
-      def gitDetails = getGitCommitDetails(7)
+      def gitDetails = getGitCommitDetails()
       executable shell
       workingDir dockerBuildDir
       args "-c", "docker buildx build ${dockerPlatform} --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${gitDetails.hash} -t ${image} --push ."
@@ -958,17 +972,13 @@ def buildTime() {
 def calculateVersion() {
   // Regex pattern for basic calendar versioning, with provision to omit patch rev
   def calVerPattern = ~/\d+\.\d+(\.\d+)?(-.*)?/
-
+  def gitDetails = getGitCommitDetails() // Adjust length as needed
   if (project.hasProperty('version') && (project.version =~ calVerPattern)) {
-    if (project.hasProperty('versionappendcommit') && project.versionappendcommit == "true") {
-      def gitDetails = getGitCommitDetails(7) // Adjust length as needed
-      return "${project.version}-${gitDetails.hash}"
-    }
-    return "${project.version}"
+    println("Generating project version using supplied version: ${project.version}-${gitDetails.hash}")
+    return "${project.version}-${gitDetails.hash}"
   } else {
     // If no version is supplied or it doesn't match the semantic versioning, calculate from git
-    println("Generating project version as supplied is version not semver: ${project.version}")
-    def gitDetails = getGitCommitDetails(7) // Adjust length as needed
+    println("Generating project version using date (${gitDetails.date}-develop-${gitDetails.hash}), as supplied version is not semver: ${project.version}")
     return "${gitDetails.date}-develop-${gitDetails.hash}"
   }
 }