[wip] spotless

.github/workflows/beam_PreCommit_Spotless.yml

@@ -62,7 +62,7 @@ env:
 jobs:
   beam_PreCommit_Spotless:
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
-    timeout-minutes: 120
+    timeout-minutes: 360
     runs-on: [self-hosted, ubuntu-24.04, main]
     strategy:
       fail-fast: false
@@ -77,6 +77,8 @@ jobs:
       github.event.comment.body == 'Run Spotless PreCommit'
     steps:
       - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
       - name: Setup repository
         uses: ./.github/actions/setup-action
         with:
@@ -85,6 +87,29 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
+      - name: Fetch base branch for Spotless ratchet
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request_target" ]; then
+            BASE_REF="${{ github.event.pull_request.base.ref }}"
+            PR_NUMBER="${{ github.event.pull_request.number }}"
+          elif [ "${{ github.event_name }}" = "issue_comment" ]; then
+            BASE_REF=$(gh pr view ${{ github.event.issue.number }} --repo ${{ github.repository }} --json baseRefName --jq .baseRefName)
+            PR_NUMBER="${{ github.event.issue.number }}"
+          else
+            BASE_REF="master"
+          fi
+          echo "Base ref is $BASE_REF"
+          if [ -n "$BASE_REF" ]; then
+            git fetch --no-tags --prune origin ${BASE_REF}:refs/remotes/origin/${BASE_REF}
+          fi
+          if [ -n "$PR_NUMBER" ]; then
+            echo "Deepening PR #$PR_NUMBER"
+            git fetch --no-tags origin +refs/pull/${PR_NUMBER}/merge:refs/remotes/pull/${PR_NUMBER}/merge || \
+            git fetch --no-tags origin +refs/pull/${PR_NUMBER}/head:refs/remotes/pull/${PR_NUMBER}/head
+          fi
+        shell: bash
       - name: run Spotless PreCommit script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:

build.gradle.kts

@@ -26,6 +26,35 @@ plugins {
   id("net.researchgate.release") version "2.8.1"
   id("org.apache.beam.module")
   id("org.sonarqube") version "3.0"
+  id("com.diffplug.spotless")
+}
+
+val mavenCentralMirrorUrl = project.findProperty("mavenCentralMirrorUrl") as String?
+val isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null
+val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank()
+
+repositories {
+  if (useMirror) {
+    maven { url = uri(mavenCentralMirrorUrl!!) }
+  } else {
+    mavenCentral()
+  }
+}
+
+spotless {
+  predeclareDeps()
+}
+
+configure<com.diffplug.gradle.spotless.SpotlessExtensionPredeclare> {
+  java {
+    googleJavaFormat("1.17.0")
+  }
+  groovy {
+    greclipse()
+  }
+  groovyGradle {
+    greclipse()
+  }
 }
 
 /*************************************************************************************************/

buildSrc/build.gradle.kts

@@ -32,7 +32,7 @@ val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank()
 plugins {
   `java-gradle-plugin`
   groovy
-  id("com.diffplug.spotless") version "5.6.1"
+  id("com.diffplug.spotless") version "7.2.1"
 }
 
 // Define the set of repositories required to fetch and enable plugins.
@@ -68,7 +68,7 @@ dependencies {
 
   runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13")                                         // Enable proto code generation
   runtimeOnly("com.github.davidmc24.gradle.plugin:gradle-avro-plugin:1.9.1")                               // Enable Avro code generation. Version 1.1.0 is the last supporting avro 1.10.2
-  runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1")                                        // Enable a code formatting plugin
+  runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:7.2.1")                                        // Enable a code formatting plugin
   runtimeOnly("gradle.plugin.com.dorongold.plugins:task-tree:1.5")                                         // Adds a 'taskTree' task to print task dependency tree
   runtimeOnly("net.linguica.gradle:maven-settings-plugin:0.5")
   runtimeOnly("gradle.plugin.io.pry.gradle.offline_dependencies:gradle-offline-dependencies-plugin:0.5.0") // Enable creating an offline repository

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -507,6 +507,41 @@ class BeamModulePlugin implements Plugin<Project> {
 
     project.ext.mavenGroupId = 'org.apache.beam'
 
+    project.ext.getRatchetBranch = {
+      ->
+      if (project.hasProperty('disableSpotlessRatchet') && project.disableSpotlessRatchet == 'true') {
+        return null
+      }
+      try {
+        def checkRef = { ref ->
+          try {
+            def res = project.exec {
+              executable 'git'
+              args 'rev-parse', '--verify', ref
+              ignoreExitValue = true
+              standardOutput = new ByteArrayOutputStream()
+              errorOutput = new ByteArrayOutputStream()
+            }
+            return res.getExitValue() == 0
+          } catch (Exception e) {
+            return false
+          }
+        }
+
+        if (checkRef('upstream/master')) {
+          return 'upstream/master'
+        } else if (checkRef('origin/master')) {
+          return 'origin/master'
+        } else if (checkRef('master')) {
+          return 'master'
+        } else {
+          return null
+        }
+      } catch (Exception e) {
+        return null
+      }
+    }
+
     // Default to dash-separated directories for artifact base name,
     // which will also be the default artifactId for maven publications
     project.apply plugin: 'base'
@@ -1312,7 +1347,10 @@ class BeamModulePlugin implements Plugin<Project> {
 
         // The "errorprone" configuration controls the classpath used by errorprone static analysis, which
         // has different dependencies than our project.
-        if (config.getName() != "errorprone" && !inDependencyUpdates) {
+        if (config.getName() != "errorprone" &&
+            !config.getName().startsWith("spotless") &&
+            !config.getName().startsWith("checkstyle") &&
+            !inDependencyUpdates) {
           config.resolutionStrategy {
             // Filtering versionless coordinates that depend on BOM. Beam project needs to set the
             // versions for only handful libraries when building the project (BEAM-9542).
@@ -1476,9 +1514,13 @@ class BeamModulePlugin implements Plugin<Project> {
           project.disableSpotlessCheck == 'true'
       project.spotless {
         enforceCheck !disableSpotlessCheck
+        def ratchetBranch = project.ext.getRatchetBranch()
+        if (ratchetBranch != null) {
+          ratchetFrom ratchetBranch
+        }
         java {
           licenseHeader javaLicenseHeader
-          googleJavaFormat('1.7')
+          googleJavaFormat('1.17.0')
           target project.fileTree(project.projectDir) {
             include 'src/*/java/**/*.java'
             exclude '**/DefaultPackageTest.java'
@@ -1602,6 +1644,8 @@ class BeamModulePlugin implements Plugin<Project> {
       project.tasks.withType(JavaCompile).configureEach {
         // we configure the Java compiler to use UTF-8.
         options.encoding = "UTF-8"
+        options.fork = true
+        options.forkOptions.memoryMaximumSize = '4g'
         // If compiled on newer JDK, set byte code compatibility
         if (requireJavaVersion.compareTo(JavaVersion.current()) < 0) {
           def compatVersion = project.javaVersion == '11' ? '11' : project.javaVersion
@@ -2098,7 +2142,8 @@ class BeamModulePlugin implements Plugin<Project> {
                     } else {
                       dependencyNode.appendNode('groupId', it.group)
                       dependencyNode.appendNode('artifactId', it.name)
-                      if (it.version != null) { // bom-managed artifacts do not have their versions
+                      if (it.version != null) {
+                        // bom-managed artifacts do not have their versions
                         dependencyNode.appendNode('version', it.version)
                       }
                       dependencyNode.appendNode('scope', param.scope)
@@ -2403,10 +2448,17 @@ class BeamModulePlugin implements Plugin<Project> {
           project.disableSpotlessCheck == 'true'
       project.spotless {
         enforceCheck !disableSpotlessCheck
+        def ratchetBranch = project.ext.getRatchetBranch()
+        if (ratchetBranch != null) {
+          ratchetFrom ratchetBranch
+        }
         def grEclipseConfig = project.project(":").file("buildSrc/greclipse.properties")
         groovy {
           greclipse().configFile(grEclipseConfig)
-          target project.fileTree(project.projectDir) { include '**/*.groovy' }
+          target project.fileTree(project.projectDir) {
+            include '**/*.groovy'
+            exclude '**/build/**'
+          }
         }
         groovyGradle { greclipse().configFile(grEclipseConfig) }
       }
@@ -2465,7 +2517,8 @@ class BeamModulePlugin implements Plugin<Project> {
       project.protobuf {
         protoc {
           // The artifact spec for the Protobuf Compiler
-          artifact = "com.google.protobuf:protoc:$protobuf_version" }
+          artifact = "com.google.protobuf:protoc:$protobuf_version"
+        }
 
         // Configure the codegen plugins
         plugins {
@@ -2547,7 +2600,8 @@ class BeamModulePlugin implements Plugin<Project> {
       project.protobuf {
         protoc {
           // The artifact spec for the Protobuf Compiler
-          artifact = "com.google.protobuf:protoc:${GrpcVendoring_1_69_0.protobuf_version}" }
+          artifact = "com.google.protobuf:protoc:${GrpcVendoring_1_69_0.protobuf_version}"
+        }
 
         // Configure the codegen plugins
         plugins {
@@ -2745,7 +2799,8 @@ class BeamModulePlugin implements Plugin<Project> {
         doLast {
           def beamPythonTestPipelineOptions = [
             "pipeline_opts": config.pythonPipelineOptions + (usesDataflowRunner ? [
-              "--sdk_location=${project.ext.sdkLocation}"]
+              "--sdk_location=${project.ext.sdkLocation}"
+            ]
             : []),
             "test_opts": config.pytestOptions,
             "suite": config.name,
@@ -3037,7 +3092,8 @@ class BeamModulePlugin implements Plugin<Project> {
         doLast {
           def beamPythonTestPipelineOptions = [
             "pipeline_opts": config.pythonPipelineOptions + (usesDataflowRunner ? [
-              "--sdk_location=${project.ext.sdkLocation}"]
+              "--sdk_location=${project.ext.sdkLocation}"
+            ]
             : []),
             "test_opts": config.pytestOptions,
             "suite": config.name,

runners/flink/build.gradle

@@ -18,15 +18,23 @@
 
 apply plugin: 'com.diffplug.spotless'
 
+repositories {
+  mavenCentral()
+}
+
 /*
  * This build.gradle file is empty except to configure the spotless task on the java sources.
  * These java sources are included in the subproject's sources in order to compile against the different versions.
  * We configure spotless here once in order to only check these java sources once.
  */
 spotless {
+  def ratchetBranch = project.rootProject.hasProperty('getRatchetBranch') ? project.rootProject.ext.getRatchetBranch() : null
+  if (ratchetBranch != null) {
+    ratchetFrom ratchetBranch
+  }
   java {
     licenseHeader org.apache.beam.gradle.BeamModulePlugin.javaLicenseHeader
-    googleJavaFormat('1.7')
+    googleJavaFormat('1.17.0')
     target project.fileTree(project.projectDir) { include 'src/*/java/**/*.java' }
   }
 }

runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/WorkerStatusPagesTest.java

@@ -38,16 +38,18 @@ public class WorkerStatusPagesTest {
   private final Server server = new Server();
   private final LocalConnector connector = new LocalConnector(server);
   @Mock private MemoryMonitor mockMemoryMonitor;
+  private boolean healthy = true;
 
   private boolean mockHealthyIndicator() {
-    return true;
+    return healthy;
   }
 
   private WorkerStatusPages wsp;
 
   @Before
   public void setUp() throws Exception {
     MockitoAnnotations.initMocks(this);
+    healthy = true;
     wsp = new WorkerStatusPages(server, mockMemoryMonitor, this::mockHealthyIndicator);
     server.addConnector(connector);
     wsp.start();
@@ -77,10 +79,7 @@ public void testHealthzHealthy() throws Exception {
 
   @Test
   public void testHealthzUnhealthy() throws Exception {
-    // set up WorkerStatusPages that respond unhealthy status on "healthz"
-    wsp.stop();
-    wsp = new WorkerStatusPages(server, mockMemoryMonitor, () -> false);
-    wsp.start();
+    healthy = false;
 
     String response = getPage("/healthz");
     assertThat(response, containsString("HTTP/1.1 500 Server Error"));

runners/spark/build.gradle

@@ -18,15 +18,23 @@
 
 apply plugin: 'com.diffplug.spotless'
 
+repositories {
+  mavenCentral()
+}
+
 /*
  * This build.gradle file is empty except to configure the spotless task on the java sources.
  * These java sources are included in the subproject's sources in order to compile against the different versions.
  * We configure spotless here once in order to only check these java sources once.
  */
 spotless {
+  def ratchetBranch = project.rootProject.hasProperty('getRatchetBranch') ? project.rootProject.ext.getRatchetBranch() : null
+  if (ratchetBranch != null) {
+    ratchetFrom ratchetBranch
+  }
   java {
     licenseHeader org.apache.beam.gradle.BeamModulePlugin.javaLicenseHeader
-    googleJavaFormat('1.7')
+    googleJavaFormat('1.17.0')
     target project.fileTree(project.projectDir) { include 'src/*/java/**/*.java' }
   }
 }

sdks/java/io/cassandra/build.gradle

@@ -62,7 +62,7 @@ dependencies {
 configurations.all (Configuration it) -> {
   // error-prone requires newer guava, don't override for annotation processing
   // https://github.com/google/error-prone/issues/2745
-  if (it.name == "annotationProcessor" || it.name =="testAnnotationProcessor") {
+  if (it.name == "annotationProcessor" || it.name =="testAnnotationProcessor" || it.name.startsWith("spotless") || it.name.startsWith("checkstyle")) {
     return
   }
   resolutionStrategy {

sdks/java/io/hadoop-file-system/build.gradle

@@ -85,7 +85,7 @@ hadoopVersions.each {kv ->
 configurations.all (Configuration it) -> {
   // error-prone requires newer guava, don't override for annotation processing
   // https://github.com/google/error-prone/issues/2745
-  if (it.name == "annotationProcessor" || it.name =="testAnnotationProcessor") {
+  if (it.name == "annotationProcessor" || it.name =="testAnnotationProcessor" || it.name.startsWith("spotless") || it.name.startsWith("checkstyle")) {
     return
   }
   resolutionStrategy {

sdks/java/io/hadoop-format/build.gradle

@@ -125,7 +125,7 @@ hadoopVersions.each {kv ->
 configurations.all (Configuration it) -> {
   // error-prone requires newer guava, don't override for annotation processing
   // https://github.com/google/error-prone/issues/2745
-  if (it.name == "annotationProcessor" || it.name =="testAnnotationProcessor") {
+  if (it.name == "annotationProcessor" || it.name =="testAnnotationProcessor" || it.name.startsWith("spotless") || it.name.startsWith("checkstyle")) {
     return
   }
   resolutionStrategy {