Merge branch 'master' into flink-stable-input-drain-bug

.asf.yaml

@@ -40,12 +40,75 @@ github:
     - olehborysevych
     - rshamunov
     - andreydevyatkin
+    - liferoad
 
   enabled_merge_buttons:
     squash: true
     merge: true
     rebase: false
 
+  protected_branches:
+    master: {}
+    release-2.51.0: {}
+    release-2.50.0: {}
+    release-2.49.0: {}
+    release-2.48.0: {}
+    release-2.47.0: {}
+    release-2.46.0: {}
+    release-2.45.0: {}
+    release-2.44.0: {}
+    release-2.43.0: {}
+    release-2.42.0: {}
+    release-2.41.0: {}
+    release-2.40.0: {}
+    release-2.39.0: {}
+    release-2.38.0: {}
+    release-2.37.0: {}
+    release-2.36.0: {}
+    release-2.35.0: {}
+    release-2.34.0: {}
+    release-2.33.0: {}
+    release-2.32.0: {}
+    release-2.31.0: {}
+    release-2.30.0: {}
+    release-2.29.0: {}
+    release-2.28.0: {}
+    release-2.27.0: {}
+    release-2.26.0: {}
+    release-2.25.0: {}
+    release-2.24.0: {}
+    release-2.23.0: {}
+    release-2.22.0: {}
+    release-2.21.0: {}
+    release-2.20.0: {}
+    release-2.19.0: {}
+    release-2.18.0: {}
+    release-2.17.0: {}
+    release-2.16.0: {}
+    release-2.15.0: {}
+    release-2.14.0: {}
+    release-2.13.0: {}
+    release-2.12.0: {}
+    release-2.11.0: {}
+    release-2.10.0: {}
+    release-2.8.0: {}
+    release-2.8.0: {}
+    release-2.7.0: {}
+    release-2.6.0: {}
+    release-2.5.0: {}
+    release-2.4.0: {}
+    release-2.3.0: {}
+    release-2.2.0: {}
+    release-2.1.1: {}
+    release-2.1.0: {}
+    release-0.6.0: {}
+    release-0.5.0: {}
+    release-0.4.0: {}
+    release-0.4.0-incubating: {}
+    release-0.3.0-incubating: {}
+    release-0.2.0-incubating: {}
+    release-0.1.0-incubating: {}
+
 notifications:
   commits: [email protected]
   issues: [email protected]

.github/ISSUE_TEMPLATE/bug.yml

@@ -63,6 +63,7 @@ body:
        - label: "Component: Go SDK"
        - label: "Component: Typescript SDK"
        - label: "Component: IO connector"
+       - label: "Component: Beam YAML"
        - label: "Component: Beam examples"
        - label: "Component: Beam playground"
        - label: "Component: Beam katas"

.github/ISSUE_TEMPLATE/failing_test.yml

@@ -69,6 +69,7 @@ body:
        - label: "Component: Go SDK"
        - label: "Component: Typescript SDK"
        - label: "Component: IO connector"
+       - label: "Component: Beam YAML"
        - label: "Component: Beam examples"
        - label: "Component: Beam playground"
        - label: "Component: Beam katas"

.github/ISSUE_TEMPLATE/feature.yml

@@ -57,6 +57,7 @@ body:
        - label: "Component: Go SDK"
        - label: "Component: Typescript SDK"
        - label: "Component: IO connector"
+       - label: "Component: Beam YAML"
        - label: "Component: Beam examples"
        - label: "Component: Beam playground"
        - label: "Component: Beam katas"

.github/ISSUE_TEMPLATE/task.yml

@@ -58,6 +58,7 @@ body:
        - label: "Component: Go SDK"
        - label: "Component: Typescript SDK"
        - label: "Component: IO connector"
+       - label: "Component: Beam YAML"
        - label: "Component: Beam examples"
        - label: "Component: Beam playground"
        - label: "Component: Beam katas"

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,7 +8,7 @@ Thank you for your contribution! Follow this checklist to help us incorporate yo
  - [ ] Update `CHANGES.md` with noteworthy changes.
  - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf).
 
-See the [Contributor Guide](https://beam.apache.org/contribute) for more tips on [how to make review process smoother](https://beam.apache.org/contribute/get-started-contributing/#make-the-reviewers-job-easier).
+See the [Contributor Guide](https://beam.apache.org/contribute) for more tips on [how to make review process smoother](https://github.com/apache/beam/blob/master/CONTRIBUTING.md#make-the-reviewers-job-easier).
 
 To check the build health, please visit [https://github.com/apache/beam/blob/master/.test-infra/BUILD_STATUS.md](https://github.com/apache/beam/blob/master/.test-infra/BUILD_STATUS.md)
 
@@ -19,4 +19,4 @@ GitHub Actions Tests Status (on master branch)
 [![Java tests](https://github.com/apache/beam/workflows/Java%20Tests/badge.svg?branch=master&event=schedule)](https://github.com/apache/beam/actions?query=workflow%3A%22Java+Tests%22+branch%3Amaster+event%3Aschedule)
 [![Go tests](https://github.com/apache/beam/workflows/Go%20tests/badge.svg?branch=master&event=schedule)](https://github.com/apache/beam/actions?query=workflow%3A%22Go+tests%22+branch%3Amaster+event%3Aschedule)
 
-See [CI.md](https://github.com/apache/beam/blob/master/CI.md) for more information about GitHub Actions CI.
+See [CI.md](https://github.com/apache/beam/blob/master/CI.md) for more information about GitHub Actions CI or the [workflows README](https://github.com/apache/beam/blob/master/.github/workflows/README.md) to see a list of phrases to trigger workflows.

.github/REVIEWERS.yml

@@ -45,15 +45,17 @@ labels:
     reviewers:
       - chamikaramj
       - johnjcasey
-      - pabloem
       - Abacn
       - ahmedabu98
       - bvolpato
-      - manavgarg
     exclusionList: []
   - name: spanner
     reviewers:
       - nielm
+  - name: bigtable
+    reviewers:
+      - igorbernstein2
+      - mutianf
     exclusionList: []
   - name: Build
     reviewers:

.github/actions/gradle-command-self-hosted-action/action.yml

@@ -24,12 +24,6 @@ inputs:
     required: false
     description: 'Gradle options'
     default: ''
-  default-arguments:
-    required: false
-    description: 'Default gradle switches' # Copied from CommonJobProperties.groovy'
-    default: |
-      --continue -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g \
-      -Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses
   max-workers:
     required: false
     description: 'Max number of workers'
@@ -44,6 +38,9 @@ runs:
         # Removing settings.xml is a workaround to avoid a decryption issue
         # of Beam's gradle-command-action plugin and github's provided
         # maven settings.xml file
-        rm ~/.m2/settings.xml
-        ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} ${{ inputs.arguments }} \
-        ${{ inputs.default-arguments }}
+        if [ -f ~/.m2/settings.xml ]; then
+          rm ~/.m2/settings.xml
+        fi
+        ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} --continue \
+         -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g -Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses \
+        ${{ inputs.arguments }}

.github/actions/rerun-job-action/action.yml

@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+#Action used to trigger a failed check re-run within a PR using a comment. Add this action to your workflow with an if condition 
+#to check if the comment is present 
+#If the check is failed this will trigger it again. If its not failed a new instance of workflow will run which will not show in the status box or checks tab in the PR and can be found in the actions tab https://github.com/apache/beam/actions
+
+name: "Rerun Job Action"
+description: Re-runs a job that is attached to the PR as Check Run
+inputs:
+  pull_request_url:
+    description: "The URL of the PR"
+    required: true
+  github_repository:
+    description: "The GitHub repository"
+    required: true
+  github_token:
+    description: "The GitHub token"
+    required: true
+  github_job:
+    description: "The GitHub job"
+    required: true
+  github_current_run_id:
+    description: "The GitHub current run id. Not the same that is fetched in this action"
+    required: true
+
+
+runs:
+  using: composite
+  steps:
+  - name: Get Last Commit SHA
+    shell: bash
+    run: | 
+      URL=${{inputs.pull_request_url}}/commits
+      PRSHA=$(curl  \
+      -H 'Authorization: Bearer ${{inputs.github_token}}' \
+      -H "Accept: application/vnd.github+json" \
+      -H "X-GitHub-Api-Version: 2022-11-28" \
+      -s $URL  | jq -r '.[-1].sha' ) 
+      echo prsha=$PRSHA >> $GITHUB_ENV
+  - name: Get Status and Conclusion for PR Job
+    shell: bash
+    run: | 
+      JOB="${{inputs.github_job}}"
+      QUERY_JOB=${JOB// /+}
+      URL="${{github.api_url}}/repos/${{inputs.github_repository}}/commits/${{env.prsha}}/check-runs?check_name=$QUERY_JOB"
+      CHECK_RUN=$(curl  \
+      -H 'Authorization: Bearer ${{inputs.github_token}}' \
+      -H "Accept: application/vnd.github+json" \
+      -H "X-GitHub-Api-Version: 2022-11-28" \
+      -s $URL | jq -r '.check_runs | .[] | select(.name=="${{inputs.github_job}}")')
+      if [ -z "$CHECK_RUN" ]; then
+        echo "No check runs found for this job"
+        echo skip=true >> $GITHUB_ENV
+        exit 0
+      fi
+      read -r STATUS CONCLUSION CHECK_SUITE_ID<<< $(echo $CHECK_RUN | jq -r '"\(.status) \(.conclusion) \(.check_suite.id)"')
+      echo status=$STATUS >> $GITHUB_ENV
+      echo conclusion=$CONCLUSION >> $GITHUB_ENV
+      echo check_suite_id=$CHECK_SUITE_ID >> $GITHUB_ENV
+ 
+
+  - name: Disable Rerun for Success or Skipped
+    if: ${{(env.status == 'completed' && (env.conclusion == 'success' || env.conclusion == 'skipped')) || env.skip == 'true'}}
+    shell: bash
+    run: echo rerun=false >> $GITHUB_ENV
+
+  - name: Get Run ID
+    if: ${{env.rerun != 'false' }}
+    shell: bash
+    run: | 
+      URL="${{github.api_url}}/repos/${{inputs.github_repository}}/actions/runs?check_suite_id=${{env.check_suite_id}}"
+      RUN_ID=$(curl  \
+      -H 'Authorization: Bearer ${{inputs.github_token}}' \
+      -H "Accept: application/vnd.github+json" \
+      -H "X-GitHub-Api-Version: 2022-11-28" \
+      -s $URL | jq -r '.workflow_runs | .[0] | .id')
+      echo run_id=$RUN_ID >> $GITHUB_ENV
+  - name: Get Job ID
+    if: ${{env.rerun != 'false' }}
+    shell: bash
+    run: | 
+      URL="${{github.api_url}}/repos/${{inputs.github_repository}}/actions/runs/${{env.run_id}}/jobs"
+      JOB_ID=$(curl  \
+      -H 'Authorization: Bearer ${{inputs.github_token}}' \
+      -H "Accept: application/vnd.github+json" \
+      -H "X-GitHub-Api-Version: 2022-11-28" \
+      -s $URL | jq -r '.jobs | .[] | select(.name=="${{inputs.github_job}}") | .id ')
+      echo job_id=$JOB_ID >> $GITHUB_ENV
+  - name: Trigger Re-run
+    if: ${{env.rerun != 'false' }}
+    shell: bash
+    run: | 
+      URL="${{github.api_url}}/repos/${{inputs.github_repository}}/actions/jobs/${{env.job_id}}/rerun"
+      curl -X POST \
+      -H 'Authorization: Bearer ${{inputs.github_token}}' \
+      -H "Accept: application/vnd.github+json" \
+      -H "X-GitHub-Api-Version: 2022-11-28" \
+      -s $URL 
+  - name: Install GH Cli
+    if: ${{env.rerun != 'false' }}
+    shell: bash
+    run: |
+      wget https://github.com/cli/cli/releases/download/v2.31.0/gh_2.31.0_linux_amd64.tar.gz
+      tar -xvf gh_2.31.0_linux_amd64.tar.gz
+      sudo mv gh_2.31.0_linux_amd64/bin/gh /usr/local/bin
+  - name: Exit rest of the run
+    if: ${{env.rerun != 'false' }}
+    shell: bash
+    run: |
+      gh run cancel ${{ inputs.github_current_run_id }}
+      gh run watch ${{ inputs.github_current_run_id }}
+    env:
+      GITHUB_TOKEN: ${{ inputs.github_token }}

.github/actions/setup-action/action.yml

@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: 'Setup action'
+description: 'Setup repository and configure the required steps'
+inputs:
+  comment_phrase:
+    description: "The comment phrase in the PR"
+    required: true
+  github_token:
+    description: "The GitHub token"
+    required: true
+  github_job:
+    description: "The GitHub job"
+    required: false
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - name: Check if the phrase is correct
+      shell: bash
+      if: github.event_name == 'issue_comment' && github.event.comment.body != inputs.comment_phrase
+      run: |
+        echo "The comment ${{ github.event.comment.body }} does not match the phrase for this instance:  ${{ inputs.comment_phrase }}. Exiting."
+        exit 1
+    - name: Check out repository code if pull request commit
+      shell: bash
+      if: ${{ github.event_name == 'pull_request_target' }}
+      run: |
+          # GitHub will automatically generate a merge commit when there are no merge conflicts.
+          # We first try to check that out, and fall back to checking out the tip of the pull request branch.
+          git fetch --depth=1 origin +refs/pull/${{ github.event.number }}/merge:refs/remotes/pull/${{ github.event.number }}/merge || \
+          git fetch --depth=1 origin +refs/pull/${{ github.event.number }}/head:refs/remotes/pull/${{ github.event.number }}/head
+          git checkout pull/${{ github.event.number }}/merge || git checkout pull/${{ github.event.number }}/head
+    - name: Check out repository code if comment phrase
+      shell: bash
+      if: ${{ github.event.comment.body == inputs.comment_phrase }}
+      run: |
+          # GitHub will automatically generate a merge commit when there are no merge conflicts.
+          # We first try to check that out, and fall back to checking out the tip of the pull request branch.
+          git fetch --depth=1 origin +refs/pull/${{ github.event.issue.number }}/merge:refs/remotes/pull/${{ github.event.issue.number }}/merge || \
+          git fetch --depth=1 origin +refs/pull/${{ github.event.issue.number }}/head:refs/remotes/pull/${{ github.event.issue.number }}/head
+          git checkout pull/${{ github.event.issue.number }}/merge || git checkout pull/${{ github.event.issue.number }}/head
+    - name: Rerun if comment phrase
+      if: ${{ github.event.comment.body == inputs.comment_phrase }}
+      uses: ./.github/actions/rerun-job-action
+      with:
+        github_token: ${{ inputs.github_token }}
+        github_job: ${{ inputs.github_job || github.job }}
+        github_repository: ${{ github.repository }}     
+        github_current_run_id: ${{ github.run_id }}
+        pull_request_url: ${{ github.event.issue.pull_request.url }}
+    ## Used for jobs that spawn docker containers and need to mount gcloud config directory
+    - name: expose gcloud path
+      shell: bash
+      run: |
+        echo KUBELET_GCLOUD_CONFIG_PATH=/var/lib/kubelet/pods/$POD_UID/volumes/kubernetes.io~empty-dir/gcloud >> $GITHUB_ENV
+    - name: Setup environment
+      uses: ./.github/actions/setup-environment-action