Merge branch 'master' into save-earliest-slot-block-in-dbvariable

CHANGELOG.md

@@ -9,5 +9,6 @@
 ### Breaking Changes
 
 ### Additions and Improvements
+- Clean up old beacon states when switching from ARCHIVE to PRUNE or MINIMAL data storage mode
 
 ### Bug Fixes

acceptance-tests/build.gradle

@@ -28,7 +28,7 @@ dependencies {
   testFixturesImplementation 'io.libp2p:jvm-libp2p'
   testFixturesImplementation 'org.apache.commons:commons-lang3'
   testFixturesImplementation 'commons-io:commons-io'
-  testFixturesImplementation 'org.apache.tuweni:tuweni-bytes'
+  testFixturesImplementation 'io.tmio:tuweni-bytes'
   testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api'
   testFixturesImplementation 'org.testcontainers:testcontainers'
   testFixturesImplementation 'org.testcontainers:junit-jupiter'

beacon/pow/build.gradle

@@ -18,7 +18,7 @@ dependencies {
 
   api 'org.web3j:core'
 
-  implementation 'org.apache.tuweni:tuweni-units'
+  implementation 'io.tmio:tuweni-units'
 
   testImplementation testFixtures(project(':infrastructure:async'))
   testImplementation testFixtures(project(':infrastructure:time'))

beacon/sync/build.gradle

@@ -16,7 +16,7 @@ dependencies {
     implementation project(':storage:api')
     implementation project(':infrastructure:events')
 
-    implementation 'org.apache.tuweni:tuweni-bytes'
+    implementation 'io.tmio:tuweni-bytes'
 
     testImplementation testFixtures(project(':ethereum:spec'))
     testImplementation testFixtures(project(':ethereum:statetransition'))

beacon/validator/build.gradle

@@ -33,8 +33,8 @@ dependencies {
   implementation project(':ethereum:json-types')
 
   implementation 'it.unimi.dsi:fastutil'
-  implementation 'org.apache.tuweni:tuweni-bytes'
-  implementation 'org.apache.tuweni:tuweni-ssz'
+  implementation 'io.tmio:tuweni-bytes'
+  implementation 'io.tmio:tuweni-ssz'
 
   testImplementation project(':infrastructure:metrics')
   testImplementation testFixtures(project(':ethereum:spec'))

build.gradle

@@ -4,6 +4,8 @@ import tech.pegasys.teku.depcheck.DepCheckPlugin
 
 import java.text.SimpleDateFormat
 
+import groovy.json.JsonSlurper
+
 import static tech.pegasys.teku.repackage.Repackage.repackage
 
 buildscript {
@@ -318,7 +320,8 @@ allprojects {
   }
 }
 
-def refTestVersion = 'v1.5.0-alpha.8'
+def nightly = System.getenv("NIGHTLY") != null
+def refTestVersion = nightly ? "nightly" : "v1.5.0-alpha.8"
 def blsRefTestVersion = 'v0.1.2'
 def slashingProtectionInterchangeRefTestVersion = 'v5.3.0'
 def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download'
@@ -329,7 +332,87 @@ def blsRefTestDownloadDir = "${buildDir}/blsRefTests/${blsRefTestVersion}"
 def slashingProtectionInterchangeRefTestDownloadDir = "${buildDir}/slashingProtectionInterchangeRefTests/${slashingProtectionInterchangeRefTestVersion}"
 def refTestExpandDir = "${project.rootDir}/eth-reference-tests/src/referenceTest/resources/consensus-spec-tests/"
 
-task downloadEthRefTests(type: Download) {
+def downloadFile(String url, String token, File outputFile) {
+  println "Download ${outputFile.getName()} (${url})"
+  def connection = new URL(url).openConnection()
+  connection.setRequestProperty("Authorization", "token ${token}")
+  connection.getInputStream().withCloseable { inputStream ->
+    outputFile.withOutputStream { outputStream ->
+      outputStream << inputStream
+    }
+  }
+}
+
+def downloadArtifacts(String repo, Long runId, String token, String downloadDir) {
+  def artifactsApiUrl = "https://api.github.com/repos/${repo}/actions/runs/${runId}/artifacts"
+  def connection = new URL(artifactsApiUrl).openConnection()
+  connection.setRequestProperty("Authorization", "token ${token}")
+  connection.setRequestProperty("Accept", "application/vnd.github.v3+json")
+
+  def response = new JsonSlurper().parse(connection.getInputStream())
+  if (response.artifacts && response.artifacts.size() > 0) {
+    response.artifacts.each { artifact ->
+      // We can skip the log file
+      if (artifact.name.contains("consensustestgen.log")) {
+        return
+      }
+
+      def fileOutput = new File(downloadDir, "${artifact.name}.zip")
+      downloadFile(artifact.archive_download_url, token, fileOutput)
+      ant.unzip(src: fileOutput, dest: downloadDir)
+      fileOutput.delete()
+    }
+    return true
+  }
+  return false
+}
+
+static def getLatestRunId(String repo, String workflow, String branch, String token) {
+  def apiUrl = "https://api.github.com/repos/${repo}/actions/workflows/${workflow}/runs?branch=${branch}&status=success&per_page=1"
+  def connection = new URL(apiUrl).openConnection()
+  connection.setRequestProperty("Authorization", "token ${token}")
+  connection.setRequestProperty("Accept", "application/vnd.github.v3+json")
+
+  // Query & parse the ID out of the response
+  def response = new JsonSlurper().parse(connection.getInputStream())
+  if (response.workflow_runs && response.workflow_runs.size() > 0) {
+    return response.workflow_runs[0].id
+  }
+  return null
+}
+
+task downloadEthRefTestsNightly {
+  doLast {
+    def repo = "ethereum/consensus-specs"
+    def workflowFileName = "generate_vectors.yml"
+    def branch = "dev"
+
+    // We need a GitHub API token to download the artifacts
+    def githubToken = System.getenv("GITHUB_TOKEN")
+    if (!githubToken) {
+      println "Error: GITHUB_TOKEN environment variable is not set"
+      return
+    }
+
+    // Get the latest workflow run ID
+    def runId = getLatestRunId(repo, workflowFileName, branch, githubToken)
+    if (!runId) {
+      println "Error: Failed to get latest run ID"
+      return
+    }
+
+    // Create the download directory
+    file(refTestDownloadDir).mkdirs()
+
+    // Download artifacts for the run
+    def success = downloadArtifacts(repo, runId, githubToken, refTestDownloadDir)
+    if (!success) {
+      println "Error: Failed to download artifacts"
+    }
+  }
+}
+
+task downloadEthRefTestsStable(type: Download) {
   src([
       "${refTestBaseUrl}/${refTestVersion}/general.tar.gz",
       "${refTestBaseUrl}/${refTestVersion}/minimal.tar.gz",
@@ -339,6 +422,14 @@ task downloadEthRefTests(type: Download) {
   overwrite false
 }
 
+task downloadEthRefTests {
+  if (nightly) {
+    dependsOn tasks.findByName("downloadEthRefTestsNightly")
+  } else {
+    dependsOn tasks.findByName("downloadEthRefTestsStable")
+  }
+}
+
 task downloadBlsRefTests(type: Download) {
   src([
       "${blsRefTestBaseUrl}/${blsRefTestVersion}/bls_tests_yaml.tar.gz"
@@ -887,7 +978,7 @@ subprojects {
     runtimeOnly 'org.apache.logging.log4j:log4j-core'
     runtimeOnly 'org.apache.logging.log4j:log4j-slf4j-impl'
 
-    testImplementation 'org.apache.tuweni:tuweni-junit'
+    testImplementation 'io.tmio:tuweni-junit'
     testImplementation 'org.assertj:assertj-core'
     testImplementation 'org.mockito:mockito-core'
     testImplementation 'org.junit.jupiter:junit-jupiter-api'

data/beaconrestapi/build.gradle

@@ -28,7 +28,7 @@ dependencies {
     implementation 'io.swagger.core.v3:swagger-annotations'
     implementation 'io.github.classgraph:classgraph'
     implementation 'io.javalin:javalin'
-    implementation 'org.apache.tuweni:tuweni-units'
+    implementation 'io.tmio:tuweni-units'
     implementation 'org.webjars:swagger-ui'
 
     testImplementation testFixtures(project(':storage'))

data/provider/build.gradle

@@ -17,7 +17,7 @@ dependencies {
 
     implementation project(':ethereum:json-types')
 
-    implementation 'org.apache.tuweni:tuweni-units'
+    implementation 'io.tmio:tuweni-units'
 
     testImplementation testFixtures(project(':ethereum:spec'))
     testImplementation testFixtures(project(':infrastructure:async'))

data/serializer/build.gradle

@@ -10,7 +10,7 @@ dependencies {
   implementation project(':infrastructure:jackson')
   implementation project(':infrastructure:async')
 
-  implementation 'org.apache.tuweni:tuweni-units'
+  implementation 'io.tmio:tuweni-units'
 
   testImplementation project(':data:provider')
   testImplementation testFixtures(project(':ethereum:spec'))

eth-benchmark-tests/build.gradle

@@ -17,10 +17,10 @@ dependencies {
   implementation testFixtures(project(':ethereum:statetransition'))
   jmhImplementation testFixtures(project(':eth-benchmark-tests'))
 
-  implementation 'org.apache.tuweni:tuweni-bytes'
+  implementation 'io.tmio:tuweni-bytes'
 
   jmhImplementation project(':infrastructure:crypto')
-  jmhImplementation 'org.apache.tuweni:tuweni-ssz'
+  jmhImplementation 'io.tmio:tuweni-ssz'
   jmhImplementation testFixtures(project(':ethereum:weaksubjectivity'))
   jmhImplementation testFixtures(project(':infrastructure:async'))
 
@@ -34,6 +34,6 @@ dependencies {
   testFixturesImplementation testFixtures(project(':ethereum:statetransition'))
   testFixturesImplementation testFixtures(project(':storage'))
 
-  testFixturesImplementation 'org.apache.tuweni:tuweni-bytes'
-  testFixturesImplementation 'org.apache.tuweni:tuweni-ssz'
+  testFixturesImplementation 'io.tmio:tuweni-bytes'
+  testFixturesImplementation 'io.tmio:tuweni-ssz'
 }

eth-reference-tests/build.gradle

@@ -24,9 +24,9 @@ dependencies {
   referenceTestImplementation 'org.hyperledger.besu:plugin-api'
   referenceTestImplementation 'com.fasterxml.jackson.core:jackson-databind'
   referenceTestImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
-  referenceTestImplementation 'org.apache.tuweni:tuweni-bytes'
-  referenceTestImplementation 'org.apache.tuweni:tuweni-junit'
-  referenceTestImplementation 'org.apache.tuweni:tuweni-ssz'
+  referenceTestImplementation 'io.tmio:tuweni-bytes'
+  referenceTestImplementation 'io.tmio:tuweni-junit'
+  referenceTestImplementation 'io.tmio:tuweni-ssz'
   referenceTestImplementation 'org.xerial.snappy:snappy-java'
 }
 

eth-tests/build.gradle

@@ -11,7 +11,7 @@ dependencies {
   implementation 'com.fasterxml.jackson.core:jackson-databind'
   implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
   implementation 'org.apache.commons:commons-text'
-  implementation 'org.apache.tuweni:tuweni-bytes'
-  implementation 'org.apache.tuweni:tuweni-units'
+  implementation 'io.tmio:tuweni-bytes'
+  implementation 'io.tmio:tuweni-units'
   implementation 'org.junit.jupiter:junit-jupiter-params'
 }

ethereum/dataproviders/build.gradle

@@ -3,7 +3,7 @@ dependencies {
     implementation project(':infrastructure:async')
     implementation project(':infrastructure:metrics')
 
-    implementation 'org.apache.tuweni:tuweni-bytes'
+    implementation 'io.tmio:tuweni-bytes'
 
     testImplementation testFixtures(project(':ethereum:spec'))
     testImplementation testFixtures(project(':ethereum:networks'))

ethereum/executionclient/build.gradle

@@ -14,7 +14,7 @@ dependencies {
   implementation project(':ethereum:events')
 
   api 'org.web3j:core'
-  implementation 'org.apache.tuweni:tuweni-units'
+  implementation 'io.tmio:tuweni-units'
   implementation 'io.jsonwebtoken:jjwt-api'
 
   runtimeOnly 'io.jsonwebtoken:jjwt-impl'

ethereum/json-types/build.gradle

@@ -5,7 +5,7 @@ dependencies {
     implementation project(':infrastructure:restapi')
     implementation project(':infrastructure:http')
 
-    implementation 'org.apache.tuweni:tuweni-units'
+    implementation 'io.tmio:tuweni-units'
 
     testImplementation testFixtures(project(':ethereum:spec'))
 }

ethereum/networks/build.gradle

@@ -7,7 +7,7 @@ dependencies {
     implementation project(':infrastructure:io')
     implementation project(':infrastructure:exceptions')
     implementation project(':infrastructure:time')
-    implementation 'org.apache.tuweni:tuweni-units'
+    implementation 'io.tmio:tuweni-units'
 
     testImplementation 'tech.pegasys.discovery:discovery'
 

ethereum/signingrecord/build.gradle

@@ -1,6 +1,6 @@
 dependencies {
   implementation project(':infrastructure:yaml')
-  implementation 'org.apache.tuweni:tuweni-bytes'
+  implementation 'io.tmio:tuweni-bytes'
   implementation 'com.fasterxml.jackson.core:jackson-databind'
   implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
 }

ethereum/spec/build.gradle

@@ -1,6 +1,6 @@
 dependencies {
     api 'it.unimi.dsi:fastutil'
-    api 'org.apache.tuweni:tuweni-bytes'
+    api 'io.tmio:tuweni-bytes'
     api project(':infrastructure:bls')
     api project(':infrastructure:bytes')
     api project(':infrastructure:collections')
@@ -9,9 +9,9 @@ dependencies {
 
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
-    implementation 'org.apache.tuweni:tuweni-bytes'
-    implementation 'org.apache.tuweni:tuweni-ssz'
-    implementation 'org.apache.tuweni:tuweni-units'
+    implementation 'io.tmio:tuweni-bytes'
+    implementation 'io.tmio:tuweni-ssz'
+    implementation 'io.tmio:tuweni-units'
     implementation project(':ethereum:performance-trackers')
     implementation project(':ethereum:execution-types')
     implementation project(':ethereum:pow:api')
@@ -28,16 +28,16 @@ dependencies {
     implementation project(':infrastructure:time')
 
     testFixturesApi 'com.google.guava:guava'
-    testFixturesApi 'org.apache.tuweni:tuweni-bytes'
+    testFixturesApi 'io.tmio:tuweni-bytes'
     testFixturesApi project(':ethereum:pow:api')
     testFixturesApi project(':infrastructure:ssz')
     testFixturesApi project(':infrastructure:unsigned')
 
     testFixturesImplementation 'com.fasterxml.jackson.core:jackson-databind'
     testFixturesImplementation 'net.jqwik:jqwik'
     testFixturesImplementation 'org.apache.logging.log4j:log4j-api'
-    testFixturesImplementation 'org.apache.tuweni:tuweni-units'
-    testFixturesImplementation 'org.apache.tuweni:tuweni-ssz'
+    testFixturesImplementation 'io.tmio:tuweni-units'
+    testFixturesImplementation 'io.tmio:tuweni-ssz'
     testFixturesImplementation 'org.hyperledger.besu.internal:core'
     testFixturesImplementation 'org.hyperledger.besu.internal:config'
     testFixturesImplementation 'org.hyperledger.besu:besu-datatypes'

ethereum/statetransition/build.gradle

@@ -25,7 +25,7 @@ dependencies {
   implementation project(':storage')
   implementation project(':storage:api')
 
-  implementation 'org.apache.tuweni:tuweni-units'
+  implementation 'io.tmio:tuweni-units'
 
   testImplementation testFixtures(project(':ethereum:spec'))
   testImplementation testFixtures(project(':ethereum:networks'))

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java

@@ -499,7 +499,7 @@ private BlockBlobSidecarsTracker internalOnNewBlock(
                       .finish(
                           error ->
                               LOG.error(
-                                  "An occurred while attempting to fetch blobs via local EL"));
+                                  "An error occurred while attempting to fetch blobs via local EL"));
                 }
               }
             });
@@ -576,7 +576,7 @@ private void onFirstSeen(final SlotAndBlockRoot slotAndBlockRoot) {
                         error ->
                             LOG.warn(
                                 "Local EL blobs lookup failed: {}",
-                                ExceptionUtils.getMessage(error)))
+                                ExceptionUtils.getRootCauseMessage(error)))
                     .thenRun(() -> this.fetchMissingContentFromRemotePeers(slotAndBlockRoot)),
             fetchDelay)
         .finish(

fork-choice-tests/build.gradle

@@ -18,6 +18,6 @@ dependencies {
 
     integrationTestImplementation 'com.fasterxml.jackson.core:jackson-databind'
     integrationTestImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
-    integrationTestImplementation 'org.apache.tuweni:tuweni-bytes'
-    integrationTestImplementation 'org.apache.tuweni:tuweni-junit'
+    integrationTestImplementation 'io.tmio:tuweni-bytes'
+    integrationTestImplementation 'io.tmio:tuweni-junit'
 }

fuzz/build.gradle

@@ -5,7 +5,7 @@ dependencies {
     implementation project(':infrastructure:bls')
     implementation project(':infrastructure:ssz')
 
-    implementation 'org.apache.tuweni:tuweni-bytes'
+    implementation 'io.tmio:tuweni-bytes'
     implementation 'it.unimi.dsi:fastutil'
 
     testImplementation 'org.xerial.snappy:snappy-java'

gradle/versions.gradle

@@ -76,7 +76,7 @@ dependencyManagement {
 
     dependency 'javax.annotation:javax.annotation-api:1.3.2'
 
-    dependencySet(group: 'org.apache.tuweni', version: '2.3.1') {
+    dependencySet(group: 'io.tmio', version: '2.4.2') {
       entry 'tuweni-bytes'
       entry 'tuweni-crypto'
       entry 'tuweni-junit'