Merge branch 'trunk' into gh-KAFKA-17564

# Conflicts:
#	core/src/test/scala/unit/kafka/controller/KafkaControllerTest.scala
#	core/src/test/scala/unit/kafka/server/FinalizedFeatureChangeListenerTest.scala

.github/actions/gh-api-approve-run/action.yml

@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "Approve Workflow Run"
+description: "Approve a Workflow run that has been submitted by a non-committer"
+inputs:
+  gh-token:
+    description: "The GitHub token for use with the CLI"
+    required: true
+  repository:
+    description: "The GitHub repository"
+    required: true
+    default: "apache/kafka"
+  run_id:
+    description: "The Workflow Run ID"
+    required: true
+  pr_number:
+    description: "The Pull Request number"
+    required: true
+  commit_sha:
+    description: "The SHA of the commit the run is for"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Approve Workflow Run
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.gh-token }}
+      run: |
+        echo "Approving workflow run ${{ inputs.run_id }} for PR ${{ inputs.pr_number }} at SHA ${{ inputs.commit_sha }}";
+        gh api --method POST \
+              -H 'Accept: application/vnd.github+json' \
+              -H 'X-GitHub-Api-Version: 2022-11-28' \
+              /repos/${{ inputs.repository }}/actions/runs/${{ inputs.run_id }}/approve

.github/actions/gh-api-update-status/action.yml

@@ -19,6 +19,8 @@
 name: "Update Commit Status Check"
 description: "Update the status of a commit check using the GH CLI"
 inputs:
+  # Composite actions do not support typed parameters. Everything is treated as a string
+  # See: https://github.com/actions/runner/issues/2238
   gh-token:
     description: "The GitHub token for use with the CLI"
     required: true

.github/actions/setup-gradle/action.yml

@@ -19,6 +19,8 @@
 name: "Gradle Setup"
 description: "Setup Java and Gradle"
 inputs:
+  # Composite actions do not support typed parameters. Everything is treated as a string
+  # See: https://github.com/actions/runner/issues/2238
   java-version:
     description: "Java version to use"
     default: "17"

.github/configs/labeler.yml

@@ -0,0 +1,124 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+build:
+  - changed-files:
+    - any-glob-to-any-file:
+      - '.github/**'
+      - 'gradle/**'
+      - '*gradlew*'
+
+clients:
+- changed-files:
+  - any-glob-to-any-file:
+    - 'clients/**'
+
+
+tests:
+- all:
+  - changed-files:
+    - any-glob-to-all-files: 
+      - 'tests/**'
+      - '**/test/**'
+
+
+connect:
+- all:
+  - changed-files:
+    - any-glob-to-any-file: 
+      - 'connect/**/*'
+    - all-globs-to-all-files:
+      - '!connect/mirror/*'
+
+mirror-maker-2:
+- changed-files:
+  - any-glob-to-any-file: 
+    - 'connect/mirror/**'
+
+
+core:
+- changed-files:
+  - any-glob-to-any-file:
+    - 'core/**'
+    - '*server*/**'
+
+storage:
+- changed-files:
+  - any-glob-to-any-file:
+    - 'storage/**'
+
+streams:
+- changed-files:
+  - any-glob-to-any-file:
+    - 'streams/**'
+
+tools:
+- changed-files:
+  - any-glob-to-any-file:
+    - 'tools/**'
+    - 'trogdor/**'
+    - 'vagrant/**'
+    - 'committer-tools/**'
+    - 'log4j-appender/**'
+    - 'shell/**'
+
+docs:
+- changed-files:
+  - any-glob-to-all-files:
+    - 'docs/**'
+    - '**/*.md'
+    - 'examples/**'
+
+generator:
+  - changed-files:
+      - any-glob-to-any-file:
+        - 'generator/**'
+
+transactions:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'transaction-coordinator/**'
+
+KIP-932:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'share/**'
+          - 'share-coordinator/**'
+
+docker:
+- changed-files:
+  - any-glob-to-any-file: 
+    - 'docker/**'
+
+performance:
+- changed-files:
+  - any-glob-to-any-file: 
+    - 'jmh-benchmarks/**'
+
+consumer:
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'clients/src/main/java/org/apache/kafka/clients/consumer/**'
+
+producer:
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'clients/src/main/java/org/apache/kafka/clients/producer/**'
+
+kraft:
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'metadata/**'
+      - 'raft/**'

.github/scripts/junit.py

@@ -160,7 +160,7 @@ def pretty_time_duration(seconds: float) -> str:
     args = parser.parse_args()
 
     reports = glob(pathname=args.path, recursive=True)
-    logger.debug(f"Found {len(reports)} JUnit results")
+    logger.info(f"Found {len(reports)} JUnit results")
     workspace_path = get_env("GITHUB_WORKSPACE") # e.g., /home/runner/work/apache/kafka
 
     total_file_count = 0
@@ -177,6 +177,7 @@ def pretty_time_duration(seconds: float) -> str:
     flaky_table = []
     skipped_table = []
 
+    logger.debug(f"::group::Parsing {len(reports)} JUnit Report Files")
     for report in reports:
         with open(report, "r") as fp:
             logger.debug(f"Parsing {report}")
@@ -215,6 +216,7 @@ def pretty_time_duration(seconds: float) -> str:
                     simple_class_name = skipped_test.class_name.split(".")[-1]
                     logger.debug(f"Found skipped test: {skipped_test}")
                     skipped_table.append((simple_class_name, skipped_test.test_name))
+    logger.debug("::endgroup::")
     duration = pretty_time_duration(total_time)
     logger.info(f"Finished processing {len(reports)} reports")
 

.github/workflows/README.md

@@ -6,35 +6,25 @@ The entry point for our build is the "CI" workflow which is defined in ci.yml.
 This is used for both PR and trunk builds. The jobs and steps of the workflow
 are defined in build.yml.
 
-## Opting-in to GitHub Actions
-
-To opt-in to the new GitHub actions workflows, simply name your branch with a
-prefix of "gh-". For example, `gh-KAFKA-17433-deflake`
+For Pull Requests, the "CI" workflow runs in an unprivileged context. This means
+it does not have access to repository secrets. After the "CI" workflow is complete, 
+the "CI Complete" workflow is automatically run. This workflow consumes artifacts
+from the "CI" workflow and does run in a privileged context. This is how we are
+able to upload Gradle Build Scans to Develocity without exposing our access
+token to the Pull Requests.
 
 ## Disabling Email Notifications
 
 By default, GitHub sends an email for each failed action run. To change this,
 visit https://github.com/settings/notifications and find System -> Actions.
 Here you can change your notification preferences.
 
-## Publishing Build Scans
-
-> This only works for committers (who have ASF accounts on ge.apache.org).
-
-There are two ways committers can have build scans published. The simplest
-way is to push their branches to apache/kafka. This will allow GitHub Actions to
-have access to the repository secret needed to publish the scan.
-
-Alternatively, committers create pull requests against their own forks and
-configure their own access key as a repository secret.
+## GitHub Actions Quirks
 
-Log in to https://ge.apache.org/, click on My Settings and then Access Keys
+### Composite Actions
 
-Generate an Access Key and give it a name like "github-actions". Copy the key
-down somewhere safe.
+Composite actions are a convenient way to reuse build logic, but they have
+some limitations. 
 
-On your fork of apache/kafka, navigate to Settings -> Security -> Secrets and
-Variables -> Actions. In the Secrets tab, click Create a New Repository Secret.
-The name of the secret should be `GE_ACCESS_TOKEN` and the value should
-be `ge.apache.org=abc123` where "abc123" is substituted for the Access Key you
-previously generated.
+- Cannot run more than one step in a composite action (see `workflow_call` instead)
+- Inputs can only be strings, no support for typed parameters. See: https://github.com/actions/runner/issues/2238

.github/workflows/build.yml

@@ -20,16 +20,16 @@ on:
     inputs:
       gradle-cache-read-only:
         description: "Should the Gradle cache be read-only?"
-        default: "true"
-        type: string
+        default: true
+        type: boolean
       gradle-cache-write-only:
         description: "Should the Gradle cache be write-only?"
-        default: "false"
-        type: string
+        default: false
+        type: boolean
       is-public-fork:
         description: "Is this CI run from a public fork?"
-        default: "true"
-        type: string
+        default: true
+        type: boolean
 
 jobs:
   validate:
@@ -66,7 +66,7 @@ jobs:
         # --no-scan:      For public fork PRs, we won't attempt to publish the scan
         run: |
           ./gradlew --build-cache --info  \
-          ${{ inputs.is-public-fork == 'true' && '--no-scan' || '--scan' }} \
+          ${{ inputs.is-public-fork && '--no-scan' || '--scan' }} \
           check -x test
       - name: Archive check reports
         if: always()

.github/workflows/ci-requested.yml

@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: CI Requested
+
+on:
+  workflow_run:
+    workflows: [CI]
+    types:
+      - requested
+
+jobs:
+  check-pr-labels:
+    # Even though job conditionals are difficult to debug, this will reduce the number of unnecessary runs
+    if: |
+      github.event_name == 'workflow_run' && 
+      github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.status == 'completed' && 
+      github.event.workflow_run.conclusion == 'action_required'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Env
+        run: printenv
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          persist-credentials:
+            false
+      - name: Check PR Labels
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
+          HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
+          HEAD_REPO: ${{ github.event.workflow_run.head_repository.owner.login }}
+        # Caution! This is a bit hacky. The GH documentation shows that the workflow_run event should include a list
+        # of referencing pull_requests. I think this might only be the case for pull requests originating from the
+        # base repository. To deal with fork PRs, we need to query the API for PRs for the owner's branch. This
+        # code assumes that the fork repo owner is the same as the organization for the "org:branch" syntax used
+        # in the query. Also, only the first matching PR from that org will be considered.
+        run: |
+          set +e
+          PR_NUMBER=$(gh api \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/pulls?head=$HEAD_REPO:$HEAD_BRANCH \
+            --jq '.[0].number')
+          if [ -z "$PR_NUMBER" ]; then
+            echo "Could not find the PR that triggered this workflow request";
+            exit 1;
+          fi
+          gh pr view $PR_NUMBER --json labels -q '.labels[].name' | grep -q 'ci-approved'
+          exitcode="$?"
+          if [ $exitcode -ne 0 ]; then
+            echo "No ci-approved label set on PR #$PR_NUMBER. Will not auto-approve.";
+            exit 0;
+          else
+            echo "Found 'ci-approved' label on PR #$PR_NUMBER. Auto-approving workflow run $RUN_ID.";
+          fi
+          echo "PR_NUMBER=$PR_NUMBER" >> "$GITHUB_ENV"
+          echo "RUN_ID=$RUN_ID" >> "$GITHUB_ENV"
+      - name: Approve Workflow Run
+        if: env.RUN_ID != ''
+        uses: ./.github/actions/gh-api-approve-run
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+          repository: ${{ github.repository }}
+          run_id: ${{ env.RUN_ID }}
+          pr_number: ${{ env.PR_NUMBER }}
+          commit_sha: ${{ github.event.workflow_run.head_sha }}

.github/workflows/ci.yml

@@ -35,6 +35,6 @@ jobs:
     with:
       gradle-cache-read-only: ${{ github.ref != 'refs/heads/trunk' }}
       gradle-cache-write-only: ${{ github.ref == 'refs/heads/trunk' }}
-      is-public-fork: ${{ github.event.pull_request.head.repo.fork == 'true' }}
+      is-public-fork: ${{ github.event.pull_request.head.repo.fork || false }}
     secrets:
       inherit