diff --git a/.github/workflows/beam_PreCommit_Spotless.yml b/.github/workflows/beam_PreCommit_Spotless.yml index 64701930e856..e8a124621f42 100644 --- a/.github/workflows/beam_PreCommit_Spotless.yml +++ b/.github/workflows/beam_PreCommit_Spotless.yml @@ -62,7 +62,7 @@ env: jobs: beam_PreCommit_Spotless: name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - timeout-minutes: 120 + timeout-minutes: 360 runs-on: [self-hosted, ubuntu-24.04, main] strategy: fail-fast: false @@ -77,6 +77,8 @@ jobs: github.event.comment.body == 'Run Spotless PreCommit' steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 - name: Setup repository uses: ./.github/actions/setup-action with: @@ -85,6 +87,29 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + - name: Fetch base branch for Spotless ratchet + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ "${{ github.event_name }}" = "pull_request_target" ]; then + BASE_REF="${{ github.event.pull_request.base.ref }}" + PR_NUMBER="${{ github.event.pull_request.number }}" + elif [ "${{ github.event_name }}" = "issue_comment" ]; then + BASE_REF=$(gh pr view ${{ github.event.issue.number }} --repo ${{ github.repository }} --json baseRefName --jq .baseRefName) + PR_NUMBER="${{ github.event.issue.number }}" + else + BASE_REF="master" + fi + echo "Base ref is $BASE_REF" + if [ -n "$BASE_REF" ]; then + git fetch --no-tags --prune origin ${BASE_REF}:refs/remotes/origin/${BASE_REF} + fi + if [ -n "$PR_NUMBER" ]; then + echo "Deepening PR #$PR_NUMBER" + git fetch --no-tags origin +refs/pull/${PR_NUMBER}/merge:refs/remotes/pull/${PR_NUMBER}/merge || \ + git fetch --no-tags origin +refs/pull/${PR_NUMBER}/head:refs/remotes/pull/${PR_NUMBER}/head + fi + shell: bash - name: run Spotless PreCommit script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/build.gradle.kts b/build.gradle.kts index 4af8fa3f1ab4..fd999aecacf4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -26,6 +26,35 @@ plugins { id("net.researchgate.release") version "2.8.1" id("org.apache.beam.module") id("org.sonarqube") version "3.0" + id("com.diffplug.spotless") +} + +val mavenCentralMirrorUrl = project.findProperty("mavenCentralMirrorUrl") as String? +val isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null +val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank() + +repositories { + if (useMirror) { + maven { url = uri(mavenCentralMirrorUrl!!) } + } else { + mavenCentral() + } +} + +spotless { + predeclareDeps() +} + +configure { + java { + googleJavaFormat("1.17.0") + } + groovy { + greclipse() + } + groovyGradle { + greclipse() + } } /*************************************************************************************************/ diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 96b1cb12dd4c..ad75eff8f6b1 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -32,7 +32,7 @@ val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank() plugins { `java-gradle-plugin` groovy - id("com.diffplug.spotless") version "5.6.1" + id("com.diffplug.spotless") version "7.2.1" } // Define the set of repositories required to fetch and enable plugins. @@ -68,7 +68,7 @@ dependencies { runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13") // Enable proto code generation runtimeOnly("com.github.davidmc24.gradle.plugin:gradle-avro-plugin:1.9.1") // Enable Avro code generation. Version 1.1.0 is the last supporting avro 1.10.2 - runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1") // Enable a code formatting plugin + runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:7.2.1") // Enable a code formatting plugin runtimeOnly("gradle.plugin.com.dorongold.plugins:task-tree:1.5") // Adds a 'taskTree' task to print task dependency tree runtimeOnly("net.linguica.gradle:maven-settings-plugin:0.5") runtimeOnly("gradle.plugin.io.pry.gradle.offline_dependencies:gradle-offline-dependencies-plugin:0.5.0") // Enable creating an offline repository diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 34132bc8149d..58a77377e2f6 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -507,6 +507,41 @@ class BeamModulePlugin implements Plugin { project.ext.mavenGroupId = 'org.apache.beam' + project.ext.getRatchetBranch = { + -> + if (project.hasProperty('disableSpotlessRatchet') && project.disableSpotlessRatchet == 'true') { + return null + } + try { + def checkRef = { ref -> + try { + def res = project.exec { + executable 'git' + args 'rev-parse', '--verify', ref + ignoreExitValue = true + standardOutput = new ByteArrayOutputStream() + errorOutput = new ByteArrayOutputStream() + } + return res.getExitValue() == 0 + } catch (Exception e) { + return false + } + } + + if (checkRef('upstream/master')) { + return 'upstream/master' + } else if (checkRef('origin/master')) { + return 'origin/master' + } else if (checkRef('master')) { + return 'master' + } else { + return null + } + } catch (Exception e) { + return null + } + } + // Default to dash-separated directories for artifact base name, // which will also be the default artifactId for maven publications project.apply plugin: 'base' @@ -1312,7 +1347,10 @@ class BeamModulePlugin implements Plugin { // The "errorprone" configuration controls the classpath used by errorprone static analysis, which // has different dependencies than our project. - if (config.getName() != "errorprone" && !inDependencyUpdates) { + if (config.getName() != "errorprone" && + !config.getName().startsWith("spotless") && + !config.getName().startsWith("checkstyle") && + !inDependencyUpdates) { config.resolutionStrategy { // Filtering versionless coordinates that depend on BOM. Beam project needs to set the // versions for only handful libraries when building the project (BEAM-9542). @@ -1476,9 +1514,13 @@ class BeamModulePlugin implements Plugin { project.disableSpotlessCheck == 'true' project.spotless { enforceCheck !disableSpotlessCheck + def ratchetBranch = project.ext.getRatchetBranch() + if (ratchetBranch != null) { + ratchetFrom ratchetBranch + } java { licenseHeader javaLicenseHeader - googleJavaFormat('1.7') + googleJavaFormat('1.17.0') target project.fileTree(project.projectDir) { include 'src/*/java/**/*.java' exclude '**/DefaultPackageTest.java' @@ -1602,6 +1644,8 @@ class BeamModulePlugin implements Plugin { project.tasks.withType(JavaCompile).configureEach { // we configure the Java compiler to use UTF-8. options.encoding = "UTF-8" + options.fork = true + options.forkOptions.memoryMaximumSize = '4g' // If compiled on newer JDK, set byte code compatibility if (requireJavaVersion.compareTo(JavaVersion.current()) < 0) { def compatVersion = project.javaVersion == '11' ? '11' : project.javaVersion @@ -2098,7 +2142,8 @@ class BeamModulePlugin implements Plugin { } else { dependencyNode.appendNode('groupId', it.group) dependencyNode.appendNode('artifactId', it.name) - if (it.version != null) { // bom-managed artifacts do not have their versions + if (it.version != null) { + // bom-managed artifacts do not have their versions dependencyNode.appendNode('version', it.version) } dependencyNode.appendNode('scope', param.scope) @@ -2403,10 +2448,17 @@ class BeamModulePlugin implements Plugin { project.disableSpotlessCheck == 'true' project.spotless { enforceCheck !disableSpotlessCheck + def ratchetBranch = project.ext.getRatchetBranch() + if (ratchetBranch != null) { + ratchetFrom ratchetBranch + } def grEclipseConfig = project.project(":").file("buildSrc/greclipse.properties") groovy { greclipse().configFile(grEclipseConfig) - target project.fileTree(project.projectDir) { include '**/*.groovy' } + target project.fileTree(project.projectDir) { + include '**/*.groovy' + exclude '**/build/**' + } } groovyGradle { greclipse().configFile(grEclipseConfig) } } @@ -2465,7 +2517,8 @@ class BeamModulePlugin implements Plugin { project.protobuf { protoc { // The artifact spec for the Protobuf Compiler - artifact = "com.google.protobuf:protoc:$protobuf_version" } + artifact = "com.google.protobuf:protoc:$protobuf_version" + } // Configure the codegen plugins plugins { @@ -2547,7 +2600,8 @@ class BeamModulePlugin implements Plugin { project.protobuf { protoc { // The artifact spec for the Protobuf Compiler - artifact = "com.google.protobuf:protoc:${GrpcVendoring_1_69_0.protobuf_version}" } + artifact = "com.google.protobuf:protoc:${GrpcVendoring_1_69_0.protobuf_version}" + } // Configure the codegen plugins plugins { @@ -2745,7 +2799,8 @@ class BeamModulePlugin implements Plugin { doLast { def beamPythonTestPipelineOptions = [ "pipeline_opts": config.pythonPipelineOptions + (usesDataflowRunner ? [ - "--sdk_location=${project.ext.sdkLocation}"] + "--sdk_location=${project.ext.sdkLocation}" + ] : []), "test_opts": config.pytestOptions, "suite": config.name, @@ -3037,7 +3092,8 @@ class BeamModulePlugin implements Plugin { doLast { def beamPythonTestPipelineOptions = [ "pipeline_opts": config.pythonPipelineOptions + (usesDataflowRunner ? [ - "--sdk_location=${project.ext.sdkLocation}"] + "--sdk_location=${project.ext.sdkLocation}" + ] : []), "test_opts": config.pytestOptions, "suite": config.name, diff --git a/runners/flink/build.gradle b/runners/flink/build.gradle index 4d058a481820..10f1afa79b23 100644 --- a/runners/flink/build.gradle +++ b/runners/flink/build.gradle @@ -18,15 +18,23 @@ apply plugin: 'com.diffplug.spotless' +repositories { + mavenCentral() +} + /* * This build.gradle file is empty except to configure the spotless task on the java sources. * These java sources are included in the subproject's sources in order to compile against the different versions. * We configure spotless here once in order to only check these java sources once. */ spotless { + def ratchetBranch = project.rootProject.hasProperty('getRatchetBranch') ? project.rootProject.ext.getRatchetBranch() : null + if (ratchetBranch != null) { + ratchetFrom ratchetBranch + } java { licenseHeader org.apache.beam.gradle.BeamModulePlugin.javaLicenseHeader - googleJavaFormat('1.7') + googleJavaFormat('1.17.0') target project.fileTree(project.projectDir) { include 'src/*/java/**/*.java' } } } \ No newline at end of file diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/WorkerStatusPagesTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/WorkerStatusPagesTest.java index 980d71ffda8c..afa2970f645c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/WorkerStatusPagesTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/WorkerStatusPagesTest.java @@ -38,9 +38,10 @@ public class WorkerStatusPagesTest { private final Server server = new Server(); private final LocalConnector connector = new LocalConnector(server); @Mock private MemoryMonitor mockMemoryMonitor; + private boolean healthy = true; private boolean mockHealthyIndicator() { - return true; + return healthy; } private WorkerStatusPages wsp; @@ -48,6 +49,7 @@ private boolean mockHealthyIndicator() { @Before public void setUp() throws Exception { MockitoAnnotations.initMocks(this); + healthy = true; wsp = new WorkerStatusPages(server, mockMemoryMonitor, this::mockHealthyIndicator); server.addConnector(connector); wsp.start(); @@ -77,10 +79,7 @@ public void testHealthzHealthy() throws Exception { @Test public void testHealthzUnhealthy() throws Exception { - // set up WorkerStatusPages that respond unhealthy status on "healthz" - wsp.stop(); - wsp = new WorkerStatusPages(server, mockMemoryMonitor, () -> false); - wsp.start(); + healthy = false; String response = getPage("/healthz"); assertThat(response, containsString("HTTP/1.1 500 Server Error")); diff --git a/runners/spark/build.gradle b/runners/spark/build.gradle index 4d058a481820..10f1afa79b23 100644 --- a/runners/spark/build.gradle +++ b/runners/spark/build.gradle @@ -18,15 +18,23 @@ apply plugin: 'com.diffplug.spotless' +repositories { + mavenCentral() +} + /* * This build.gradle file is empty except to configure the spotless task on the java sources. * These java sources are included in the subproject's sources in order to compile against the different versions. * We configure spotless here once in order to only check these java sources once. */ spotless { + def ratchetBranch = project.rootProject.hasProperty('getRatchetBranch') ? project.rootProject.ext.getRatchetBranch() : null + if (ratchetBranch != null) { + ratchetFrom ratchetBranch + } java { licenseHeader org.apache.beam.gradle.BeamModulePlugin.javaLicenseHeader - googleJavaFormat('1.7') + googleJavaFormat('1.17.0') target project.fileTree(project.projectDir) { include 'src/*/java/**/*.java' } } } \ No newline at end of file diff --git a/sdks/java/io/cassandra/build.gradle b/sdks/java/io/cassandra/build.gradle index 5256ac1f2148..e41b8a9144d0 100644 --- a/sdks/java/io/cassandra/build.gradle +++ b/sdks/java/io/cassandra/build.gradle @@ -62,7 +62,7 @@ dependencies { configurations.all (Configuration it) -> { // error-prone requires newer guava, don't override for annotation processing // https://github.com/google/error-prone/issues/2745 - if (it.name == "annotationProcessor" || it.name =="testAnnotationProcessor") { + if (it.name == "annotationProcessor" || it.name =="testAnnotationProcessor" || it.name.startsWith("spotless") || it.name.startsWith("checkstyle")) { return } resolutionStrategy { diff --git a/sdks/java/io/hadoop-file-system/build.gradle b/sdks/java/io/hadoop-file-system/build.gradle index d78d6a300cac..96aef57bdbba 100644 --- a/sdks/java/io/hadoop-file-system/build.gradle +++ b/sdks/java/io/hadoop-file-system/build.gradle @@ -85,7 +85,7 @@ hadoopVersions.each {kv -> configurations.all (Configuration it) -> { // error-prone requires newer guava, don't override for annotation processing // https://github.com/google/error-prone/issues/2745 - if (it.name == "annotationProcessor" || it.name =="testAnnotationProcessor") { + if (it.name == "annotationProcessor" || it.name =="testAnnotationProcessor" || it.name.startsWith("spotless") || it.name.startsWith("checkstyle")) { return } resolutionStrategy { diff --git a/sdks/java/io/hadoop-format/build.gradle b/sdks/java/io/hadoop-format/build.gradle index 73fc44a0f311..86a713c5bd30 100644 --- a/sdks/java/io/hadoop-format/build.gradle +++ b/sdks/java/io/hadoop-format/build.gradle @@ -125,7 +125,7 @@ hadoopVersions.each {kv -> configurations.all (Configuration it) -> { // error-prone requires newer guava, don't override for annotation processing // https://github.com/google/error-prone/issues/2745 - if (it.name == "annotationProcessor" || it.name =="testAnnotationProcessor") { + if (it.name == "annotationProcessor" || it.name =="testAnnotationProcessor" || it.name.startsWith("spotless") || it.name.startsWith("checkstyle")) { return } resolutionStrategy {